Spaces:
Paused
Paused
| # -*- coding: utf-8 -*- | |
| # MarkUp Language | |
| import zipOrZip as oz | |
| import mw; from mw import * | |
| import re,os | |
| import urllib.request | |
| from urllib.parse import urlparse,quote,quote_plus,unquote,unquote_plus,urlencode | |
| orror=mw.orro3("roml.txt"); | |
| urltmpdir="r:\\TEMP\\" | |
| hp="https://" | |
| class webrap: | |
| user_agents = [ | |
| 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', | |
| 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36', | |
| ## 'Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.129 Mobile Safari/537.36' | |
| 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11', | |
| 'Opera/9.25 (Windows NT 5.1; U; en)', | |
| 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', | |
| 'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)', | |
| 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12' | |
| 'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9' | |
| ] | |
| headers=[] | |
| headers.append( ('User-agent', user_agents[0])) | |
| headers.append((b'\x41\x63\x63\x65\x70\x74-\x65\x6E\x63\x6F\x64\x69\x6E\x67'.decode(), 'gzip'))# encoding | |
| def __init__(zl): | |
| zl.user_agent=webrap.user_agents[0]; | |
| zl.code="utf-8"; | |
| def gZip(zl): | |
| byte = zl.httpres.read() | |
| hs={f.lower():v for f,v in zl.httpres.getheaders()} | |
| if('content-encoding' in hs and hs['content-encoding'] == 'gzip'): | |
| byte = oz.extgz(byte); | |
| return byte | |
| def g0(zl): | |
| opener = urllib.request.build_opener() # | |
| opener.addheaders = zl.headers | |
| #print(zl.url) | |
| zl.httpres = opener.open(zl.url)#,timeout=2) | |
| return zl.gZip(); | |
| def getCache(zl):#offline cache | |
| esurl=zl.url | |
| esurl=mw.esfn(esurl.replace(hp,"")) | |
| # esurl=unquote_plus(esurl,zl.code) | |
| zl.offfile=urltmpdir+esurl | |
| ## zl.offfile=zl.offfile[:255] | |
| if(zl.url.find("://")==-1): | |
| zl.url=hp+zl.url | |
| if(mw.ifexist(zl.offfile)): | |
| byte=mw.rfdb(zl.offfile) | |
| zl.byte=byte; | |
| return zl.byte | |
| return None; | |
| def orwgetof0(zl): | |
| if(not zl.getCache()): | |
| ## zl.url=quote_plus(zl.url,encoding=code,safe=":/") | |
| zl.byte = zl.g0(); | |
| wfdb(zl.offfile,zl.byte); orRemove(zl.offfile, rembat="wget cache remove.bat") | |
| return zl.byte; | |
| def orwgetoff(url,post=0): | |
| zl=webrap() | |
| zl.url=url; | |
| return zl.orwgetof0() | |
| def utf(zl,url): | |
| zl.url=url; | |
| t1=zl.orwgetof0(); | |
| t2=t1.decode(zl.code); | |
| return t2; | |
| def gton(zl,*aw): return zl.gtoff(*aw,on=True) | |
| def gtoff(zl,url,code="",on=False): | |
| if "#" in url: url=url.split("#")[0] | |
| zl.url=quote(unquote_plus(url),safe=":/?=") | |
| ## print(zl.url) | |
| zl.code=code; | |
| t1=zl.g0() if on else zl.orwgetof0(); | |
| if code==bytes: return t1 | |
| t2=t1.decode(code) if code else vcs(t1); | |
| return t2; | |
| def wget(zl,*aw):return zl.gtoff(*aw,on=False) | |
| # | |
| def orwget(zl,url,post=0,off=1):#legacy | |
| esurl=url | |
| esurl=esfn(esurl.replace(hp,"")) | |
| esurl=unquote_plus(esurl,encoding=zl.code) | |
| zl.offfile=urltmpdir+esurl | |
| if(url.find(hp)==-1): | |
| url=hp+url | |
| if(ifexist(zl.offfile) and off): | |
| byte=rfdb(zl.offfile) | |
| else: | |
| ## url=quote_plus(url,encoding=code,safe=":/") | |
| user_agent=webrap.user_agents[0]; | |
| opener = urllib.request.build_opener() | |
| opener.addheaders = [ ('User-agent', user_agent), ('Accept-encoding', 'gzip')] | |
| qpost=zl.mkpost(post) if post else None# | |
| zl.res = opener.open(url,qpost.encode("utf8"))#zl.res | |
| byte = zl.res.read() | |
| hs={k:v for k,v in zl.res.getheaders()} | |
| if('Content-Encoding' in hs and hs['Content-Encoding'] == 'gzip'): | |
| byte = oz.extgz(byte); | |
| if off: wfdb(zl.offfile,byte) | |
| return byte.decode(zl.code); | |
| def mkpost(zl,q): | |
| return jw("&",[jw("=",[quote_plus(str(w),encoding=zl.code) for w in f]) for f in q]) | |
| ## return tea | |
| ## l = [] | |
| if 1: | |
| tea="" | |
| for k, v in q: | |
| k = quote_plus(str(k),encoding=zl.code) | |
| v = quote_plus(str(v),encoding=zl.code) | |
| #l.append(k + '=' + v) | |
| tea+=k+"="+v+"&" | |
| return tea[:-1] | |
| irle=re.compile(r'<a href="(?P<href>.+?)".*?>(?P<te>.*?)</a>'); | |
| def le(te): | |
| "link extracter" | |
| zl.links=webrap.irle.findall(te); | |
| return zl.links; | |
| def nx(zl,reg,te,url,page=1): | |
| "pager" | |
| #par=urlparse(zl.url)#netloc path query | |
| irnx=re.compile(reg) | |
| def fpnxurl(tefp): | |
| if not(mo:=irnx.search(tefp)):print("not match regnx"); return | |
| else: | |
| if type(page)!=int: | |
| nxquery=mo.group(1) | |
| nxquery=nxquery.replace("&","&") | |
| nxurl=jw("------",url, nxquery) | |
| print(nxquery) | |
| ## print((zl.url, nxquery), nxurl) | |
| return nxurl | |
| return True | |
| nxurl=url | |
| if te: | |
| yield te | |
| if type(page)==int: page+=1 | |
| nxurl=fpnxurl(te) | |
| for n in range(page or 0,987987987): | |
| if page: nxurl=url.format(n) | |
| ## print(nxurl) | |
| te=zl.gtoff(nxurl,zl.code, on=1) | |
| yield te | |
| if not(nxurl:=fpnxurl(te)): break | |
| def firefox(zl,url): | |
| import webbrowser | |
| webbrowser.open(url) | |
| def ggle(zl,q): | |
| zl.code="utf-8" | |
| zl.code="shift-jis" | |
| post=(("q",q),("hl","ja"))#japan | |
| qp=zl.mkpost(post) | |
| q2="http://www.google.co.jp/search?"+qp | |
| te=zl.orwget(q2) | |
| return te | |
| def chr(sl,te):lwt | |
| ## 文字参照や実体参照を通常の文字に戻す ［Python, Tips］ - Programming Magic5 | |
| # 実体参照 & 文字参照を通常の文字に戻す | |
| def htmlentity2unicode(text): | |
| import html.entities as htmlentitydefs | |
| # 正規表現のコンパイル | |
| reference_regex = re.compile('&(#x?[0-9a-f]+|[a-z]+);', re.IGNORECASE) | |
| num16_regex = re.compile('#x\d+', re.IGNORECASE) | |
| num10_regex = re.compile('#\d+', re.IGNORECASE) | |
| result = "" | |
| i = 0 | |
| while True: | |
| # 実体参照 or 文字参照を見つける | |
| match = reference_regex.search(text, i) | |
| if match is None: | |
| result += text[i:] | |
| break | |
| result += text[i:match.start()] | |
| i = match.end() | |
| name = match.group(1) | |
| # 実体参照 | |
| if name in htmlentitydefs.name2codepoint.keys(): | |
| result += chr(htmlentitydefs.name2codepoint[name]) | |
| # 文字参照 | |
| elif num16_regex.match(name): | |
| # 16進数 | |
| result += chr(int('0'+name[1:], 16)) | |
| elif num10_regex.match(name): | |
| # 10進数 | |
| result += chr(int(name[1:])) | |
| return result | |
| if 0: | |
| webrap().ggle("かんこれ"); | |
| def yaho(zl,q): | |
| zl.code="utf-8" | |
| post=(("ei","UTF-8"),("p",q),("n","40"))#nPageNum10|15|20|30|40|100 | |
| qp=zl.mkpost(post) | |
| q2="http://search.yahoo.co.jp/search?"+qp | |
| te=zl.orwget(q2) | |
| return te | |
| def bing(zl,url): | |
| esurl=url | |
| esurl=esfn(esurl.replace(hp,"")) | |
| zl.code=code="utf-8" | |
| esurl=unquote_plus(esurl,encoding=code) | |
| zl.offfile=urltmpdir+esurl | |
| if(url.find(hp)==-1): | |
| url=hp+url | |
| if(ifexist(zl.offfile)): | |
| byte=rfdb(zl.offfile) | |
| else: | |
| #url=quote_plus(url,encoding=code,safe=":/") | |
| user_agent=webrap.user_agents[0]; | |
| opener = urllib.request.build_opener() | |
| ## opener.addheaders = [ ('','')] | |
| opener.addheaders= [('Host',' www.bing.com')] | |
| opener.addheaders= [('User-Agent',' Mozilla/5.0 (Windows NT 5.1; rv:26.0) Gecko/20100101 Firefox/26.0')] | |
| opener.addheaders= [('Accept',' text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')] | |
| opener.addheaders= [('Accept-Language',' ja,en-us;q=0.7,en;q=0.3')] | |
| opener.addheaders= [('Accept-Encoding',' gzip, deflate')] | |
| qpost=zl.mkpost(post) if post else None# | |
| zl.res = opener.open(url)#zl.res | |
| byte = zl.res.read() | |
| hs={k:v for k,v in zl.res.getheaders()}#header had raised TypeError | |
| if('Content-Encoding' in hs and hs['Content-Encoding'] == 'gzip'): | |
| byte = oz.extgz(byte); | |
| wfdb(zl.offfile,byte) | |
| return byte.decode(code); | |
| def quotesafe(zl,url): | |
| qurl=quote_plus(url,encoding=code,safe=":/&?") | |
| write8(r"r:\url1",url) | |
| write8(r"r:\url2",qurl) | |
| if 0:#getoff | |
| im=webrap() | |
| im.url=r"http://blog.livedoor.jp/yuriss/" | |
| te=im.orwgetoff(im.url) | |
| wfdb(r"r:\get.txt",te); | |
| #ftp | |
| from ftplib import FTP | |
| class owf: | |
| def login(sl,url,aka,pwd): | |
| sl.imf=FTP(url) | |
| sl.imf.login(aka,pwd) | |
| def list(sl): | |
| return sl.imf.retrlines("LIST") | |
| def cbO(sl,blockData): | |
| sl.res+=blockData#packet | |
| def o(sl,fn): | |
| sl.res=b"" | |
| ri=sl.imf.retrbinary('RETR '+fn,sl.cbO) | |
| return sl.res | |
| def uf(sl,fn): | |
| with open(fn, "rb")as fo: | |
| #_ftp.cwd("/target_dir/") | |
| return sl.imf.storbinary("STOR "+os.path.split(fn)[1], fo) | |
| def u(sl,fn,fo):#file object | |
| return sl.imf.storbinary("STOR "+fn, fo) | |
| def dele(sl): | |
| sl.imf.quit() | |
| #------------ | |
| ##from __future__ import print_function | |
| from threading import Thread,Lock,Event,Condition | |
| import socket | |
| from contextlib import closing | |
| import select | |
| defaultPort=4000 | |
| class orListen(Thread): | |
| #python listen | |
| def __init__(sl,cb=None,fil=(),ercb=(),**kw): | |
| super().__init__(**kw) | |
| sl.owari=False | |
| if cb: sl._cb=cb | |
| sl.fil=fil | |
| sl.ercb=ercb or (sl._ercb, []) | |
| # | |
| sl.port = defaultPort | |
| def i4l(sl,port,cb=None,fil=(),ercb=(),**kw): | |
| super().__init__(**kw) | |
| sl.owari=False | |
| assert sl._cb, "reimmpriment cb" | |
| sl.fil=fil | |
| sl.ercb=(sl._ercb, []) | |
| # | |
| sl.host = '127.0.0.1' | |
| sl.port = port | |
| return sl | |
| def main(sl):#, port = defaultPort): | |
| backlog = 10 | |
| bufsize = 4096 | |
| server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| sl.server_sock=server_sock | |
| Set = set([server_sock]) | |
| try: | |
| server_sock.bind((sl.host, sl.port)) | |
| server_sock.listen(backlog) | |
| while True: | |
| rready, wready, xready = select.select(Set, [], [],5.0) | |
| ## print("ready") | |
| if sl.owari: | |
| print("owari select") | |
| break | |
| for sock in rready: | |
| try: | |
| if sock is server_sock: | |
| conn, address = server_sock.accept() | |
| Set.add(conn)#peer | |
| else: | |
| te = sock.recv(bufsize) | |
| if len(te) == 0:pass | |
| elif te==b"owari": | |
| sock.send(b"len-0;") | |
| Set.remove(sock) | |
| sl.owari=1 | |
| shuttown(sock) | |
| else: | |
| sl.sock=sock | |
| sl._cb(te, sl.fil) | |
| except Exception as e: | |
| except2(edit=1) | |
| Set.remove(sock) | |
| shuttown(sock) | |
| #sl.ercb[0](e,sl.ercb[1]) | |
| except Exception as e: | |
| print(e) | |
| except2(edit=1) | |
| #sl.ercb[0](e,sl.ercb[1]) | |
| finally: | |
| for sock in Set: | |
| shuttown(sock) | |
| return | |
| def run(sl):#start | |
| sl.main() | |
| ## except ConnectionResetError as er: | |
| def chkbind(sl): | |
| server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| Set = set([server_sock]) | |
| try: | |
| server_sock.bind((sl.host, sl.port)) | |
| ## server_sock.listen(backlog) | |
| except Exception as e: | |
| e2=traceback.format_exc() | |
| rn=0 | |
| else: | |
| rn=1 | |
| finally: | |
| for sock in Set: | |
| shuttown(sock) | |
| return rn | |
| def Release(sl):#finalist | |
| if sl.chkbind(): return | |
| send1("127.0.0.1", port=sl.port, te="owari")#4 unblock select | |
| sl.owari=True | |
| class _orListen(orListen): | |
| def _cb(sl,te,fil): | |
| te=te.decode() | |
| print(te) | |
| if(te=="x"): sl.Release() | |
| #sl.sock.send(te.encode()) | |
| sl.cb2(te,fil) | |
| ## _cb=cb | |
| def cb2(sl,te,fil): | |
| pn=sl.sock.getpeername() | |
| ## print(te) | |
| so=sl.sock | |
| if 2:#local | |
| lcl=dict(locals()) | |
| lcl.update({"r":"r:"}) | |
| try: | |
| exec(te,globals(),lcl) | |
| except Exception as e: | |
| print("exec:", traceback.format_exc()) | |
| for i in dict(locals()): lcl.pop(i,55) | |
| #sl.sock.lcl=lcl | |
| # | |
| sl.sock.send(lcl["r"].encode()) | |
| def _ercb(sl,erte,fil): | |
| print("---",erte) | |
| def chkbind(port): | |
| return _orListen().i4l(port).chkbind() | |
| def send1(host, port, te): | |
| sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| with closing(sock): | |
| sock.connect((host, port)) | |
| sock.send(te.encode("utf8")) | |
| def mainClient(fil=(0,),sp=(),iLi=(),host = '127.0.0.1',port = defaultPort,bufsize = 4096,timeout=2.0): | |
| let=[] | |
| once,=fil | |
| sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| with closing(sock): | |
| sock.connect((host, port)) | |
| if iLi: | |
| for te in iLi: | |
| try: | |
| sock.send(te.encode('utf-8')) | |
| sock.settimeout(timeout) | |
| #let.append(sock.recv(bufsize)) | |
| if 1: | |
| for x1 in range(99): | |
| te=sock.recv(bufsize) | |
| if(te):let.append(te) | |
| #print(let) | |
| except Exception as e: | |
| e2=traceback.format_exc() | |
| print(e2) | |
| return let | |
| if once: | |
| for n in range(once): | |
| try: | |
| sock.send(str(n).encode('utf-8')) | |
| print(" send;",sock.recv(bufsize)) | |
| except Exception as e: | |
| e2=traceback.format_exc() | |
| print(e2) | |
| while not once: | |
| line = sys.stdin.readline().rstrip() | |
| if len(line) == 0: | |
| print("send x to quit.") | |
| break | |
| sock.send(line.encode('utf-8')) | |
| print(" client:",sock.recv(bufsize)) | |
| return | |
| send=mainClient | |
| if __name__ == '__main__': | |
| from subprocess import * | |
| import time | |
| import sys | |
| import traceback | |
| hh,fl=hhfl([]) | |
| hh=hhww() | |
| ## hh.getaddrinfo=1 | |
| # | |
| if hh.getaddrinfo: | |
| print(socket.getaddrinfo("localhost",4000)) | |
| print(socket.getaddrinfo("localhost",8080)) | |
| # | |
| ## post=[3:4,5:6] | |
| post=[(1,1),(2,2),(3,3),("q","検索")] | |
| q=urlencode(post) | |
| print(q) | |
| im=webrap() | |
| im.code="cp932" | |
| ## print(im.mkpost(post)) | |
| if 0:#? | |
| rn=webrap().yaho("そのだうみ") | |
| ## print(rn) | |
| rn=webrap.orwgetoff("http://search.yahoo.co.jp/search?") | |
| rn=webrap().bing("http://www.bing.com/search?q=かんこれ&qs=n&form=QBLH&filt=all&pq=かんこれ&sc=8-4&sp=-1&sk=")#404repair by delete quote process | |
| rn=webrap.orwgetoff("http://livedoor-search.naver.jp/search?c=ld_blog_sb&o_st=livedoor&sm=ldb_top&q=かよちん&search_btn=検索") | |
| webrap().bing(r"http://livedoor-search.naver.jp/search?c=ld_blog_sb&o_st=livedoor&sm=ldb_top&q=かよちん&search_btn=検索"); | |
| if {"ftp"}.issubset(hh): | |
| imf=owf() | |
| # imf.login("ftp:shinuno.comxa.com","a6091821","heikou2zero")#exc | |
| imf.login("red.ribbon.to","shinuno","forget")#exc | |
| if {"ftp","ftp1"}.issubset(hh): | |
| #all stdout | |
| li=imf.list() | |
| print("----",li) | |
| li=imf.imf.retrlines("NLST") | |
| print("----",li) | |
| li=imf.imf.retrlines("MLSD") | |
| print("----",li) | |
| def mkBody(te): | |
| t2="""<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"> | |
| <html> | |
| <head> | |
| <title>Index of /video</title> | |
| </head> | |
| <body> | |
| %s | |
| </body></html> | |
| """%(te) | |
| return t2 | |
| if {"ftp","ftp2"}.issubset(hh): | |
| imf.imf.cwd("/public_html/video") | |
| print(imf.imf.pwd()) | |
| print(imf.imf.nlst()) | |
| #sendcmd("LIST") | |
| # | |
| from io import BytesIO | |
| import time | |
| t=time.localtime(); | |
| t2=("d_dd_d"%(t.tm_mday,t.tm_hour,t.tm_min,t.tm_sec)) | |
| fn="|"+t2+".txt" | |
| fnh="|"+t2+".htm" | |
| #no fn932=fn.encode("cp932") | |
| ri=imf.u(fn,BytesIO(b"fwfwf")) | |
| ri | |
| ri=imf.u(fnh,BytesIO(mkBody(time.strftime("%Y_%m/%d %H%M %A %B %c")).encode("utf8"))) | |
| if {"ftp","ftp3"}.issubset(hh): | |
| imf.imf.cwd("/public_html/cgi") | |
| # | |
| fn=r"N:\afilen\program\html\award\cgi\Dirr1.php" | |
| fn=r"N:\afilen\program\html\award\cgi\wget.php" | |
| ri=imf.uf(fn) | |
| ## dn=r"N:\afilen\program\html\award\cgi" | |
| ## for fn in os.listdir(dn):imf.uf(jofn(dn,fn)) | |
| ## tes2() | |
| ## tes1() | |
| imf.dele() | |
| import jk987 | |
| # | |
| iml=_orListen() | |
| if "client"==arps.f:#? | |
| try: | |
| ## q8erw | |
| mainClient([2]) | |
| except ConnectionRefusedError as er: | |
| print(" connectEr: ", str(er)) | |
| except Exception as e: | |
| e2=traceback.format_exc() | |
| print(e2) | |
| input("owari\n") | |
| elif "listen" in hh: | |
| iml.start() | |
| ## Popen(["E:\Python34\python.exe",sys.argv[0]]+["-f","client"],shell=True) | |
| ## Popen(["start","E:\Python34\python.exe "+jw(" ",[sys.argv[0],"-f","client"])],shell=True) | |
| ## Popen(["start",r"E:\Python34\python.exe",r"N:\afilen\program\python\qt\ClipProcess2.pyw"],shell=True)#ok | |
| ## Popen(["start",r"N:\afilen\program\python\qt\ClipProcess2.pyw"],shell=True)#ok | |
| for _ in range(4): | |
| Popen(["start",sys.argv[0], "-f","client"],shell=True) | |
| for f in range(2): | |
| print("listen:",f) | |
| time.sleep(2.0) | |
| iml.owari=True | |
| if "clien2"==arps.f:# | |
| try: | |
| mainClient([None]) | |
| except Exception as e: | |
| e2=traceback.format_exc() | |
| print(e2) | |
| input("owari\n") | |
| elif "ser2" in hh: | |
| for i in range(2): | |
| Popen(["start",sys.argv[0], "-f","clien2"],shell=True) | |
| iml.main() | |
| for i in range(2): | |
| print("listen:",i) | |
| time.sleep(2.0) | |
| iml.owari=True | |
| #sys.exit() | |
| # | |