Update extra/etc/dl.py

extra/etc/dl.py

@@ -1,19 +1,58 @@
 #!/usr/bin/env python3
-import os,re,sys,urllib.request,urllib.error
-r,g,y,c,z='\033[31m','\033[32m','\033[33m','\033[36m','\033[0m'
-die=lambda m,n=1:(print(f'{r}error:{z} {m}',file=sys.stderr),sys.exit(n))
-len(sys.argv)in(2,3)or die(f'usage: {sys.argv[0]} <url> [outdir]')
-u=sys.argv[1];d=sys.argv[2]if len(sys.argv)>2 else'.';os.makedirs(d,exist_ok=True)
-m=re.match(r'^https?://github\.com/([^/]+)/([^/]+)/blob/([^/]+)/(.*)$',u);u=f'https://raw.githubusercontent.com/{m[1]}/{m[2]}/{m[3]}/{m[4]}'if m else u
-m=re.match(r'^(https?://[^/]+/.+)/(src|blob)/([^/]+)/(.*)$',u);u=f'{m[1]}/raw/{m[3]}/{m[4]}'if m else u
-n=re.sub(r'_+','_',re.sub(r'[^a-zA-Z0-9._+-]+','_',((u.split('?',1)[0].split('#',1)[0].rstrip('/').split('/')[-1]or'downloaded_file').replace('%20','_').replace(' ','_')))).strip('._')or'downloaded_file'
-p=os.path.join(d,n);b,e=os.path.splitext(n);i=1
-while os.path.exists(p):p=os.path.join(d,f'{b}_{i}{e}');i+=1
-try:
- q=urllib.request.Request(u,headers={'User-Agent':'pyraw/1.0'})
- with urllib.request.urlopen(q,timeout=20)as r1,open(p,'wb')as f:f.write(r1.read())
- print(f'{g}saved:{z} {c}{p}{z}')
-except urllib.error.HTTPError as x:die(f'http {x.code} for {y}{u}{z}',2)
-except urllib.error.URLError as x:die(f'network failure: {x.reason}',3)
-except PermissionError:die(f'cannot write to {y}{d}{z}',4)
-except Exception as x:die(str(x),5)
+import os,re,sys,argparse,urllib.request,urllib.error
+PRESETS={'glo':'https://git.backend-3.com/dev/glo'}
+R,G,Y,C,Z='\033[31m','\033[32m','\033[33m','\033[36m','\033[0m'
+die=lambda m,n=1:(print(f'{R}error:{Z} {m}',file=sys.stderr),sys.exit(n))
+warn=lambda m:print(f'{Y}warn:{Z} {m}',file=sys.stderr)
+ok=lambda m:print(f'{G}{m}{Z}')
+nn=lambda s:(re.sub(r'_+','_',re.sub(r'[^a-zA-Z0-9._+-]+','_',s.replace('%20','_').replace(' ','_'))).strip('._')or'downloaded_file')
+def uq(p):
+    if not os.path.exists(p): return p
+    b,e=os.path.splitext(p);i=1
+    while os.path.exists(f'{b}_{i}{e}'): i+=1
+    return f'{b}_{i}{e}'
+def raw(u):
+    m=re.match(r'^https?://github\.com/([^/]+)/([^/]+)/blob/([^/]+)/(.*)$',u)
+    if m:return f'https://raw.githubusercontent.com/{m[1]}/{m[2]}/{m[3]}/{m[4]}'
+    m=re.match(r'^(https?://[^/]+/.+)/(src|blob)/([^/]+)/(.*)$',u)
+    return f'{m[1]}/raw/{m[3]}/{m[4]}' if m else u
+def splitrepo(s,db='main'):
+    s=s[1:]
+    if '#' in s:s,b=s.rsplit('#',1)
+    else:b=db
+    return PRESETS.get(s,s),b
+def rbase(repo,br):
+    repo=repo.rstrip('/')
+    if 'github.com/' in repo and '/blob/' not in repo and '/tree/' not in repo:
+        m=re.match(r'^https?://github\.com/([^/]+)/([^/]+?)/?$',repo)
+        if m:return f'https://raw.githubusercontent.com/{m[1]}/{m[2]}/{br}'
+    m=re.match(r'^(https?://[^/]+/.+?)(?:/(?:src|blob|raw|tree)/([^/]+)(?:/.*)?)?/?$',repo)
+    return f'{m[1]}/raw/{m[2]or br}' if m else die('could not derive raw base')
+def get(u,t):
+    q=urllib.request.Request(u,headers={'User-Agent':'pyraw/easy'})
+    with urllib.request.urlopen(q,timeout=t)as r:return r.read()
+def save(out,rel,b):
+    p=os.path.join(out,*[nn(x)for x in rel.strip('/').split('/')]);os.makedirs(os.path.dirname(p),exist_ok=True);p=uq(p)
+    with open(p,'wb')as f:f.write(b)
+    return p
+ap=argparse.ArgumentParser(description='easy raw file downloader')
+ap.add_argument('x',nargs='+',help='url or @repo followed by repo-relative paths')
+ap.add_argument('-o','--outdir',default='.')
+ap.add_argument('-b','--branch',default='main')
+ap.add_argument('-t','--timeout',type=int,default=20)
+a=ap.parse_args();os.makedirs(a.outdir,exist_ok=True);xs=a.x
+if xs[0].startswith('@'):
+    repo,br=splitrepo(xs[0],a.branch);base=rbase(repo,br);files=xs[1:] or die('missing repo-relative paths')
+    for rel in files:
+        u=f'{base.rstrip("/")}/{rel.lstrip("/")}'
+        try:b=get(u,a.timeout);p=save(a.outdir,rel,b);ok(f'saved: {C}{p}{Z}')
+        except urllib.error.HTTPError as e:warn(f'http {e.code} for {u}')
+        except urllib.error.URLError as e:warn(f'network failure for {u}: {e.reason}')
+        except Exception as e:warn(str(e))
+else:
+    for u0 in xs:
+        u=raw(u0);n=nn(u0.split('?',1)[0].split('#',1)[0].rstrip('/').split('/')[-1]or'downloaded_file')
+        try:b=get(u,a.timeout);p=save(a.outdir,n,b);ok(f'saved: {C}{p}{Z}')
+        except urllib.error.HTTPError as e:warn(f'http {e.code} for {u}')
+        except urllib.error.URLError as e:warn(f'network failure for {u}: {e.reason}')
+        except Exception as e:warn(str(e))