import zipfile,io,sys,glob,os,pickle,collections d=sys.argv[1]; year=sys.argv[2] data=collections.defaultdict(dict) # key->{date:vol} fsys={} restr=collections.Counter() for z in sorted(glob.glob(os.path.join(d,f'*_{year}_ccs_data.zip'))): zf=zipfile.ZipFile(z) for n in zf.namelist(): if not n.upper().endswith('.VOL'): continue for line in io.TextIOWrapper(zf.open(n),encoding='latin-1'): p=line.rstrip('\r\n').split('|') if p[0]!='V' or len(p)<34: continue r=p[34] if len(p)>34 else '' restr[r]+=1 if r.strip() not in ('','0'): continue try: hrs=[int(x) for x in p[10:34]] except ValueError: continue k=(p[1],p[3],p[4],p[5]); fsys[k]=p[2] date=f'{int(p[6]):04d}-{int(p[7]):02d}-{int(p[8]):02d}' data[k][date]=sum(hrs) # one record per lane per day print(z,len(data),flush=True) print(restr.most_common(10)) pickle.dump((dict(data),fsys),open(os.path.join(d,f'agg_{year}.pkl'),'wb'))