From c268e44f3141bb1b43cab71ca56617b560cc6395 Mon Sep 17 00:00:00 2001 From: dev Date: Mon, 2 Nov 2026 04:16:57 +0000 Subject: [PATCH] Update extra/etc/dl.py --- extra/etc/dl.py | 75 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/extra/etc/dl.py b/extra/etc/dl.py index bb422a7..15da479 100644 --- a/extra/etc/dl.py +++ b/extra/etc/dl.py @@ -1,19 +1,58 @@ #!/usr/bin/env python3 -import os,re,sys,urllib.request,urllib.error -r,g,y,c,z='\033[31m','\033[32m','\033[33m','\033[36m','\033[0m' -die=lambda m,n=1:(print(f'{r}error:{z} {m}',file=sys.stderr),sys.exit(n)) -len(sys.argv)in(2,3)or die(f'usage: {sys.argv[0]} [outdir]') -u=sys.argv[1];d=sys.argv[2]if len(sys.argv)>2 else'.';os.makedirs(d,exist_ok=True) -m=re.match(r'^https?://github\.com/([^/]+)/([^/]+)/blob/([^/]+)/(.*)$',u);u=f'https://raw.githubusercontent.com/{m[1]}/{m[2]}/{m[3]}/{m[4]}'if m else u -m=re.match(r'^(https?://[^/]+/.+)/(src|blob)/([^/]+)/(.*)$',u);u=f'{m[1]}/raw/{m[3]}/{m[4]}'if m else u -n=re.sub(r'_+','_',re.sub(r'[^a-zA-Z0-9._+-]+','_',((u.split('?',1)[0].split('#',1)[0].rstrip('/').split('/')[-1]or'downloaded_file').replace('%20','_').replace(' ','_')))).strip('._')or'downloaded_file' -p=os.path.join(d,n);b,e=os.path.splitext(n);i=1 -while os.path.exists(p):p=os.path.join(d,f'{b}_{i}{e}');i+=1 -try: - q=urllib.request.Request(u,headers={'User-Agent':'pyraw/1.0'}) - with urllib.request.urlopen(q,timeout=20)as r1,open(p,'wb')as f:f.write(r1.read()) - print(f'{g}saved:{z} {c}{p}{z}') -except urllib.error.HTTPError as x:die(f'http {x.code} for {y}{u}{z}',2) -except urllib.error.URLError as x:die(f'network failure: {x.reason}',3) -except PermissionError:die(f'cannot write to {y}{d}{z}',4) -except Exception as x:die(str(x),5) \ No newline at end of file +import os,re,sys,argparse,urllib.request,urllib.error +PRESETS={'glo':'https://git.backend-3.com/dev/glo'} +R,G,Y,C,Z='\033[31m','\033[32m','\033[33m','\033[36m','\033[0m' +die=lambda m,n=1:(print(f'{R}error:{Z} {m}',file=sys.stderr),sys.exit(n)) +warn=lambda m:print(f'{Y}warn:{Z} {m}',file=sys.stderr) +ok=lambda m:print(f'{G}{m}{Z}') +nn=lambda s:(re.sub(r'_+','_',re.sub(r'[^a-zA-Z0-9._+-]+','_',s.replace('%20','_').replace(' ','_'))).strip('._')or'downloaded_file') +def uq(p): + if not os.path.exists(p): return p + b,e=os.path.splitext(p);i=1 + while os.path.exists(f'{b}_{i}{e}'): i+=1 + return f'{b}_{i}{e}' +def raw(u): + m=re.match(r'^https?://github\.com/([^/]+)/([^/]+)/blob/([^/]+)/(.*)$',u) + if m:return f'https://raw.githubusercontent.com/{m[1]}/{m[2]}/{m[3]}/{m[4]}' + m=re.match(r'^(https?://[^/]+/.+)/(src|blob)/([^/]+)/(.*)$',u) + return f'{m[1]}/raw/{m[3]}/{m[4]}' if m else u +def splitrepo(s,db='main'): + s=s[1:] + if '#' in s:s,b=s.rsplit('#',1) + else:b=db + return PRESETS.get(s,s),b +def rbase(repo,br): + repo=repo.rstrip('/') + if 'github.com/' in repo and '/blob/' not in repo and '/tree/' not in repo: + m=re.match(r'^https?://github\.com/([^/]+)/([^/]+?)/?$',repo) + if m:return f'https://raw.githubusercontent.com/{m[1]}/{m[2]}/{br}' + m=re.match(r'^(https?://[^/]+/.+?)(?:/(?:src|blob|raw|tree)/([^/]+)(?:/.*)?)?/?$',repo) + return f'{m[1]}/raw/{m[2]or br}' if m else die('could not derive raw base') +def get(u,t): + q=urllib.request.Request(u,headers={'User-Agent':'pyraw/easy'}) + with urllib.request.urlopen(q,timeout=t)as r:return r.read() +def save(out,rel,b): + p=os.path.join(out,*[nn(x)for x in rel.strip('/').split('/')]);os.makedirs(os.path.dirname(p),exist_ok=True);p=uq(p) + with open(p,'wb')as f:f.write(b) + return p +ap=argparse.ArgumentParser(description='easy raw file downloader') +ap.add_argument('x',nargs='+',help='url or @repo followed by repo-relative paths') +ap.add_argument('-o','--outdir',default='.') +ap.add_argument('-b','--branch',default='main') +ap.add_argument('-t','--timeout',type=int,default=20) +a=ap.parse_args();os.makedirs(a.outdir,exist_ok=True);xs=a.x +if xs[0].startswith('@'): + repo,br=splitrepo(xs[0],a.branch);base=rbase(repo,br);files=xs[1:] or die('missing repo-relative paths') + for rel in files: + u=f'{base.rstrip("/")}/{rel.lstrip("/")}' + try:b=get(u,a.timeout);p=save(a.outdir,rel,b);ok(f'saved: {C}{p}{Z}') + except urllib.error.HTTPError as e:warn(f'http {e.code} for {u}') + except urllib.error.URLError as e:warn(f'network failure for {u}: {e.reason}') + except Exception as e:warn(str(e)) +else: + for u0 in xs: + u=raw(u0);n=nn(u0.split('?',1)[0].split('#',1)[0].rstrip('/').split('/')[-1]or'downloaded_file') + try:b=get(u,a.timeout);p=save(a.outdir,n,b);ok(f'saved: {C}{p}{Z}') + except urllib.error.HTTPError as e:warn(f'http {e.code} for {u}') + except urllib.error.URLError as e:warn(f'network failure for {u}: {e.reason}') + except Exception as e:warn(str(e)) \ No newline at end of file