mlwebOrWeb.py

# -*- coding: utf-8 -*-
# MarkUp Language
import zipOrZip as oz
import mw; from mw import *
import re,os
import urllib.request
from urllib.parse import urlparse,quote,quote_plus,unquote,unquote_plus,urlencode

orror=mw.orro3("roml.txt");
urltmpdir="r:\\TEMP\\"
hp="https://"
class webrap:
    user_agents = [
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
##        'Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.129 Mobile Safari/537.36'
        'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
        'Opera/9.25 (Windows NT 5.1; U; en)',
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
        'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
        'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12'
        'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9'
    ]
    headers=[]
    headers.append( ('User-agent', user_agents[0]))
    headers.append((b'\x41\x63\x63\x65\x70\x74-\x65\x6E\x63\x6F\x64\x69\x6E\x67'.decode(), 'gzip'))# encoding
    def __init__(zl):
        zl.user_agent=webrap.user_agents[0];
        zl.code="utf-8";
    def gZip(zl):
        byte = zl.httpres.read()
        hs={f.lower():v for f,v in zl.httpres.getheaders()}
        if('content-encoding' in hs and hs['content-encoding'] == 'gzip'):
            byte = oz.extgz(byte);
        return byte
    def g0(zl):
        opener = urllib.request.build_opener() #
        opener.addheaders = zl.headers
        #print(zl.url)
        zl.httpres = opener.open(zl.url)#,timeout=2)
        return zl.gZip();
    def getCache(zl):#offline cache
        esurl=zl.url
        esurl=mw.esfn(esurl.replace(hp,""))
#        esurl=unquote_plus(esurl,zl.code)
        zl.offfile=urltmpdir+esurl
##        zl.offfile=zl.offfile[:255]
        if(zl.url.find("://")==-1):
            zl.url=hp+zl.url
        if(mw.ifexist(zl.offfile)):
            byte=mw.rfdb(zl.offfile)
            zl.byte=byte;
            return zl.byte
        return None;
    def orwgetof0(zl):
        if(not zl.getCache()):
##            zl.url=quote_plus(zl.url,encoding=code,safe=":/")
            zl.byte = zl.g0();
            wfdb(zl.offfile,zl.byte); orRemove(zl.offfile, rembat="wget cache remove.bat")
        return zl.byte;
    def orwgetoff(url,post=0):
        zl=webrap()
        zl.url=url;
        return zl.orwgetof0()
    def utf(zl,url):
        zl.url=url;
        t1=zl.orwgetof0();
        t2=t1.decode(zl.code);
        return t2;
    def gton(zl,*aw): return zl.gtoff(*aw,on=True)
    def gtoff(zl,url,code="",on=False):
        if "#" in url: url=url.split("#")[0]
        zl.url=quote(unquote_plus(url),safe=":/?=")
##        print(zl.url)
        zl.code=code;
        t1=zl.g0() if on else zl.orwgetof0();
        if code==bytes: return t1
        t2=t1.decode(code) if code else vcs(t1);
        return t2;
    def wget(zl,*aw):return zl.gtoff(*aw,on=False)
    #
    def orwget(zl,url,post=0,off=1):#legacy
        esurl=url
        esurl=esfn(esurl.replace(hp,""))
        esurl=unquote_plus(esurl,encoding=zl.code)
        zl.offfile=urltmpdir+esurl
        if(url.find(hp)==-1):
            url=hp+url
        if(ifexist(zl.offfile) and off):
            byte=rfdb(zl.offfile)
        else:
##            url=quote_plus(url,encoding=code,safe=":/")
            user_agent=webrap.user_agents[0];
            opener = urllib.request.build_opener()
            opener.addheaders = [ ('User-agent', user_agent),  ('Accept-encoding', 'gzip')]
            qpost=zl.mkpost(post) if post else None#
            zl.res = opener.open(url,qpost.encode("utf8"))#zl.res
            byte = zl.res.read()
            hs={k:v for k,v in zl.res.getheaders()}
            if('Content-Encoding' in hs and hs['Content-Encoding'] == 'gzip'):
                byte = oz.extgz(byte);
            if off: wfdb(zl.offfile,byte)
        return byte.decode(zl.code);
    def mkpost(zl,q):
        return jw("&",[jw("=",[quote_plus(str(w),encoding=zl.code) for w in f]) for f in q])
##        return tea
##        l = []
        if 1:
            tea=""
            for k, v in q:
                k = quote_plus(str(k),encoding=zl.code)
                v = quote_plus(str(v),encoding=zl.code)
                #l.append(k + '=' + v)
                tea+=k+"="+v+"&"
            return tea[:-1]
    irle=re.compile(r'<a href="(?P<href>.+?)".*?>(?P<te>.*?)</a>');
    def le(te):
        "link extracter"
        zl.links=webrap.irle.findall(te);
        return zl.links;
    def nx(zl,reg,te,url,page=1):
        "pager"
        #par=urlparse(zl.url)#netloc path query
        irnx=re.compile(reg)
        def fpnxurl(tefp):
            if not(mo:=irnx.search(tefp)):print("not match regnx"); return
            else:
                if type(page)!=int:
                    nxquery=mo.group(1)
                    nxquery=nxquery.replace("&amp;","&")
                    nxurl=jw("------",url, nxquery)
                    print(nxquery)
##                    print((zl.url, nxquery), nxurl)
                    return nxurl
                return True
        nxurl=url
        if te:
            yield te
            if type(page)==int: page+=1
            nxurl=fpnxurl(te)
        for n in range(page or 0,987987987):
            if page: nxurl=url.format(n)
##            print(nxurl)
            te=zl.gtoff(nxurl,zl.code, on=1)
            yield te
            if not(nxurl:=fpnxurl(te)): break
    def firefox(zl,url):
        import webbrowser
        webbrowser.open(url)
    def ggle(zl,q):
        zl.code="utf-8"
        zl.code="shift-jis"
        post=(("q",q),("hl","ja"))#japan
        qp=zl.mkpost(post)
        q2="http://www.google.co.jp/search?"+qp
        te=zl.orwget(q2)
        return te
    def chr(sl,te):lwt
    ##    文字参照や実体参照を通常の文字に戻す ［Python, Tips］ - Programming Magic5
    # 実体参照 & 文字参照を通常の文字に戻す
    @staticmethod
    def htmlentity2unicode(text):
        import html.entities  as htmlentitydefs
        # 正規表現のコンパイル
        reference_regex = re.compile('&(#x?[0-9a-f]+|[a-z]+);', re.IGNORECASE)
        num16_regex = re.compile('#x\d+', re.IGNORECASE)
        num10_regex = re.compile('#\d+', re.IGNORECASE)

        result = ""
        i = 0
        while True:
            # 実体参照 or 文字参照を見つける
            match = reference_regex.search(text, i)
            if match is None:
                result += text[i:]
                break

            result += text[i:match.start()]
            i = match.end()
            name = match.group(1)

            # 実体参照
            if name in htmlentitydefs.name2codepoint.keys():
                result += chr(htmlentitydefs.name2codepoint[name])
            # 文字参照
            elif num16_regex.match(name):
                # 16進数
                result += chr(int('0'+name[1:], 16))
            elif num10_regex.match(name):
                # 10進数
                result += chr(int(name[1:]))

        return result

if 0:
    webrap().ggle("かんこれ");
    def yaho(zl,q):
        zl.code="utf-8"
        post=(("ei","UTF-8"),("p",q),("n","40"))#nPageNum10|15|20|30|40|100
        qp=zl.mkpost(post)
        q2="http://search.yahoo.co.jp/search?"+qp
        te=zl.orwget(q2)
        return te
    def bing(zl,url):
        esurl=url
        esurl=esfn(esurl.replace(hp,""))
        zl.code=code="utf-8"
        esurl=unquote_plus(esurl,encoding=code)
        zl.offfile=urltmpdir+esurl
        if(url.find(hp)==-1):
            url=hp+url
        if(ifexist(zl.offfile)):
            byte=rfdb(zl.offfile)
        else:
            #url=quote_plus(url,encoding=code,safe=":/")
            user_agent=webrap.user_agents[0];
            opener = urllib.request.build_opener()
##            opener.addheaders = [ ('','')]
            opener.addheaders= [('Host',' www.bing.com')]
            opener.addheaders= [('User-Agent',' Mozilla/5.0 (Windows NT 5.1; rv:26.0) Gecko/20100101 Firefox/26.0')]
            opener.addheaders= [('Accept',' text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')]
            opener.addheaders= [('Accept-Language',' ja,en-us;q=0.7,en;q=0.3')]
            opener.addheaders= [('Accept-Encoding',' gzip, deflate')]
            qpost=zl.mkpost(post) if post else None#
            zl.res = opener.open(url)#zl.res
            byte = zl.res.read()
            hs={k:v for k,v in zl.res.getheaders()}#header had raised TypeError
            if('Content-Encoding' in hs and hs['Content-Encoding'] == 'gzip'):
                byte = oz.extgz(byte);
            wfdb(zl.offfile,byte)
        return byte.decode(code);
def quotesafe(zl,url):
    qurl=quote_plus(url,encoding=code,safe=":/&?")
    write8(r"r:\url1",url)
    write8(r"r:\url2",qurl)
if 0:#getoff
    im=webrap()
    im.url=r"http://blog.livedoor.jp/yuriss/"
    te=im.orwgetoff(im.url)
    wfdb(r"r:\get.txt",te);
#ftp
from ftplib import FTP
class owf:
    def login(sl,url,aka,pwd):
        sl.imf=FTP(url)
        sl.imf.login(aka,pwd)
    def list(sl):
        return sl.imf.retrlines("LIST")
    def cbO(sl,blockData):
        sl.res+=blockData#packet
    def o(sl,fn):
        sl.res=b""
        ri=sl.imf.retrbinary('RETR '+fn,sl.cbO)
        return sl.res
    def uf(sl,fn):
        with open(fn, "rb")as fo:
            #_ftp.cwd("/target_dir/")
            return sl.imf.storbinary("STOR "+os.path.split(fn)[1], fo)
    def u(sl,fn,fo):#file object
            return sl.imf.storbinary("STOR "+fn, fo)
    def dele(sl):
        sl.imf.quit()
#------------
##from __future__ import print_function
from threading import Thread,Lock,Event,Condition
import socket
from contextlib import closing
import select
defaultPort=4000
class orListen(Thread):
    #python listen
    def __init__(sl,cb=None,fil=(),ercb=(),**kw):
        super().__init__(**kw)
        sl.owari=False
        if cb: sl._cb=cb
        sl.fil=fil
        sl.ercb=ercb or (sl._ercb, [])
        #
        sl.port = defaultPort
    def i4l(sl,port,cb=None,fil=(),ercb=(),**kw):
        super().__init__(**kw)
        sl.owari=False
        assert  sl._cb, "reimmpriment cb"
        sl.fil=fil
        sl.ercb=(sl._ercb, [])
        #
        sl.host = '127.0.0.1'
        sl.port = port
        return sl
    def main(sl):#, port = defaultPort):
      backlog = 10
      bufsize = 4096

      server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
      sl.server_sock=server_sock
      Set = set([server_sock])
      try:
        server_sock.bind((sl.host, sl.port))
        server_sock.listen(backlog)

        while True:
          rready, wready, xready = select.select(Set, [], [],5.0)
##          print("ready")
          if sl.owari:
            print("owari select")
            break
          for sock in rready:
            try:
                if sock is server_sock:
                  conn, address = server_sock.accept()
                  Set.add(conn)#peer
                else:
                  te = sock.recv(bufsize)
                  if len(te) == 0:pass
                  elif te==b"owari":
                    sock.send(b"len-0;")
                    Set.remove(sock)
                    sl.owari=1
                    shuttown(sock)
                  else:
                    sl.sock=sock
                    sl._cb(te, sl.fil)
            except Exception as e:
                except2(edit=1)
                Set.remove(sock)
                shuttown(sock)
                #sl.ercb[0](e,sl.ercb[1])
      except Exception as e:
        print(e)
        except2(edit=1)
        #sl.ercb[0](e,sl.ercb[1])
      finally:
        for sock in Set:
          shuttown(sock)
      return
    def run(sl):#start
        sl.main()
##        except ConnectionResetError as er:

    def chkbind(sl):
      server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
      Set = set([server_sock])
      try:
        server_sock.bind((sl.host, sl.port))
##        server_sock.listen(backlog)
      except Exception as e:
        e2=traceback.format_exc()
        rn=0
      else:
        rn=1
      finally:
        for sock in Set:
          shuttown(sock)
      return rn
    def Release(sl):#finalist
        if sl.chkbind(): return
        send1("127.0.0.1", port=sl.port, te="owari")#4 unblock select
        sl.owari=True
class _orListen(orListen):
    def _cb(sl,te,fil):
        te=te.decode()
        print(te)
        if(te=="x"): sl.Release()
        #sl.sock.send(te.encode())
        sl.cb2(te,fil)
##    _cb=cb
    def cb2(sl,te,fil):
        pn=sl.sock.getpeername()
##        print(te)
        so=sl.sock
        if 2:#local
            lcl=dict(locals())
            lcl.update({"r":"r:"})
            try:
                exec(te,globals(),lcl)
            except Exception as e:
                print("exec:", traceback.format_exc())
            for i in dict(locals()): lcl.pop(i,55)
            #sl.sock.lcl=lcl
        #
        sl.sock.send(lcl["r"].encode())
    def _ercb(sl,erte,fil):
        print("---",erte)
def chkbind(port):
    return _orListen().i4l(port).chkbind()
def send1(host, port, te):
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    with closing(sock):
        sock.connect((host, port))
        sock.send(te.encode("utf8"))
def mainClient(fil=(0,),sp=(),iLi=(),host = '127.0.0.1',port = defaultPort,bufsize = 4096,timeout=2.0):
  let=[]
  once,=fil

  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  with closing(sock):
    sock.connect((host, port))
    if iLi:
        for te in iLi:
            try:
              sock.send(te.encode('utf-8'))
              sock.settimeout(timeout)
              #let.append(sock.recv(bufsize))
              if 1:
                for x1 in range(99):
                    te=sock.recv(bufsize)
                    if(te):let.append(te)
              #print(let)
            except Exception as e:
                e2=traceback.format_exc()
                print(e2)
        return let
    if once:
        for n in range(once):
          try:
              sock.send(str(n).encode('utf-8'))
              print(" send;",sock.recv(bufsize))
          except Exception as e:
            e2=traceback.format_exc()
            print(e2)
    while not once:
      line = sys.stdin.readline().rstrip()
      if len(line) == 0:
        print("send x to quit.")
        break
      sock.send(line.encode('utf-8'))
      print(" client:",sock.recv(bufsize))
  return
send=mainClient
if __name__ == '__main__':
    from subprocess import *
    import time
    import sys
    import traceback
    hh,fl=hhfl([])
    hh=hhww()
##    hh.getaddrinfo=1
    #
    if hh.getaddrinfo:
        print(socket.getaddrinfo("localhost",4000))
        print(socket.getaddrinfo("localhost",8080))
    #
##    post=[3:4,5:6]
    post=[(1,1),(2,2),(3,3),("q","検索")]
    q=urlencode(post)
    print(q)
    im=webrap()
    im.code="cp932"
##    print(im.mkpost(post))
    if 0:#?
        rn=webrap().yaho("そのだうみ")
##        print(rn)
        rn=webrap.orwgetoff("http://search.yahoo.co.jp/search?")
        rn=webrap().bing("http://www.bing.com/search?q=かんこれ&qs=n&form=QBLH&filt=all&pq=かんこれ&sc=8-4&sp=-1&sk=")#404repair by delete quote process
        rn=webrap.orwgetoff("http://livedoor-search.naver.jp/search?c=ld_blog_sb&o_st=livedoor&sm=ldb_top&q=かよちん&search_btn=検索")
        webrap().bing(r"http://livedoor-search.naver.jp/search?c=ld_blog_sb&o_st=livedoor&sm=ldb_top&q=かよちん&search_btn=検索");
    if {"ftp"}.issubset(hh):
        imf=owf()
        # imf.login("ftp:shinuno.comxa.com","a6091821","heikou2zero")#exc
        imf.login("red.ribbon.to","shinuno","forget")#exc
        if {"ftp","ftp1"}.issubset(hh):
            #all stdout
            li=imf.list()
            print("----",li)
            li=imf.imf.retrlines("NLST")
            print("----",li)
            li=imf.imf.retrlines("MLSD")
            print("----",li)
        def mkBody(te):
            t2="""<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
    <html>
     <head>
      <title>Index of /video</title>
     </head>
     <body>
     %s
    </body></html>
    """%(te)
            return t2
        if {"ftp","ftp2"}.issubset(hh):
            imf.imf.cwd("/public_html/video")
            print(imf.imf.pwd())
            print(imf.imf.nlst())
            #sendcmd("LIST")
            #
            from io import BytesIO
            import time
            t=time.localtime();
            t2=("d_dd_d"%(t.tm_mday,t.tm_hour,t.tm_min,t.tm_sec))
            fn="|"+t2+".txt"
            fnh="|"+t2+".htm"
            #no fn932=fn.encode("cp932")
            ri=imf.u(fn,BytesIO(b"fwfwf"))
            ri
            ri=imf.u(fnh,BytesIO(mkBody(time.strftime("%Y_%m/%d %H%M %A %B %c")).encode("utf8")))
        if {"ftp","ftp3"}.issubset(hh):
            imf.imf.cwd("/public_html/cgi")
            #
            fn=r"N:\afilen\program\html\award\cgi\Dirr1.php"
            fn=r"N:\afilen\program\html\award\cgi\wget.php"
            ri=imf.uf(fn)
    ##        dn=r"N:\afilen\program\html\award\cgi"
    ##        for fn in os.listdir(dn):imf.uf(jofn(dn,fn))
    ##    tes2()
    ##    tes1()
        imf.dele()
        import jk987
    #
    iml=_orListen()
    if "client"==arps.f:#?
      try:
##        q8erw
        mainClient([2])
      except ConnectionRefusedError as er:
        print(" connectEr: ", str(er))
      except Exception as e:
        e2=traceback.format_exc()
        print(e2)
      input("owari\n")
    elif "listen" in hh:
        iml.start()
##        Popen(["E:\Python34\python.exe",sys.argv[0]]+["-f","client"],shell=True)
##        Popen(["start","E:\Python34\python.exe "+jw(" ",[sys.argv[0],"-f","client"])],shell=True)
##        Popen(["start",r"E:\Python34\python.exe",r"N:\afilen\program\python\qt\ClipProcess2.pyw"],shell=True)#ok
##        Popen(["start",r"N:\afilen\program\python\qt\ClipProcess2.pyw"],shell=True)#ok
        for _ in range(4):
            Popen(["start",sys.argv[0], "-f","client"],shell=True)
        for f in range(2):
            print("listen:",f)
            time.sleep(2.0)
        iml.owari=True
    if "clien2"==arps.f:#
      try:
        mainClient([None])
      except Exception as e:
        e2=traceback.format_exc()
        print(e2)
      input("owari\n")
    elif "ser2" in hh:
        for i in range(2):
            Popen(["start",sys.argv[0], "-f","clien2"],shell=True)
        iml.main()
        for i in range(2):
            print("listen:",i)
            time.sleep(2.0)
        iml.owari=True
        #sys.exit()
#