chipmunkmc/edl
1
0
Fork 0
mirror of https://github.com/bkerler/edl.git synced 2024-11-14 19:14:58 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve progress bar

Browse source
This commit is contained in:
Bjoern Kerler 2021-12-05 14:32:21 +01:00
parent 299c6dc636
commit 4d5cb463d2
4 changed files with 130 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

128
edlclient/Library/firehose.py
View file

@ -14,6 +14,7 @@ from threading import Thread
from edlclient.Library.utils import *
from edlclient.Library.gpt import gpt
from edlclient.Library.sparse import QCSparse
from edlclient.Library.utils import progress
try:
from edlclient.Library.Modules.init import modules
@ -62,7 +63,7 @@ class nand_partition:
np.attr2 = attr2
np.attr3 = attr3
np.which_flash = which_flash
self.partentries[np.name]=np
self.partentries[np.name] = np
return True
return False
@ -160,34 +161,6 @@ class firehose(metaclass=LogBase):
self.nandparttbl = None
self.nandpart = nand_partition(parent=self, printer=print)
def show_progress(self, prefix, pos, total, display=True):
t0 = time.time()
if pos == 0:
self.prog = 0
self.progtime = time.time()
self.progpos=pos
try:
prog = round(float(pos) / float(total) * float(100), 2)
if pos != total:
if prog > self.prog:
if display:
tdiff=t0-self.progtime
datasize=(pos-self.progpos)/1024/1024
throughput=(((datasize)/(tdiff)))
print_progress(prog, 100, prefix='Progress:',
suffix=prefix+' (Sector %d of %d) %0.2f MB/s' %
(pos // self.cfg.SECTOR_SIZE_IN_BYTES,
total // self.cfg.SECTOR_SIZE_IN_BYTES,
throughput), bar_length=50)
self.prog = prog
self.progpos = pos
self.progtime = t0
else:
if display:
print_progress(100, 100, prefix='Progress:', suffix='Complete', bar_length=50)
except:
pass
def detect_partition(self, arguments, partitionname):
fpartitions = {}
for lun in self.luns:
@ -423,6 +396,7 @@ class firehose(metaclass=LogBase):
sparseformat = True
total = sparse.getsize()
bytestowrite = total
progbar = progress(self.cfg.SECTOR_SIZE_IN_BYTES)
with open(filename, "rb") as rf:
# Make sure we fill data up to the sector size
num_partition_sectors = total // self.cfg.SECTOR_SIZE_IN_BYTES
@ -441,7 +415,7 @@ class firehose(metaclass=LogBase):
data += self.modules.addprogram()
data += f"/>\n</data>"
rsp = self.xmlsend(data, self.skipresponse)
self.show_progress("Write", 0, total, display)
progbar.show_progress(prefix="Write", pos=0, total=total, display=display)
if rsp[0]:
old = 0
while bytestowrite > 0:
@ -459,7 +433,7 @@ class firehose(metaclass=LogBase):
wdata += b"\x00" * (filllen - wlen)
self.cdc.write(wdata)
self.show_progress("Write", total-bytestowrite, total, display)
progbar.show_progress(prefix="Write", pos=total - bytestowrite, total=total, display=display)
self.cdc.write(b'')
# time.sleep(0.2)
@ -497,7 +471,8 @@ class firehose(metaclass=LogBase):
data += self.modules.addprogram()
data += f"/>\n</data>"
rsp = self.xmlsend(data, self.skipresponse)
self.show_progress("Write", 0, total, display)
progbar = progress(self.cfg.SECTOR_SIZE_IN_BYTES)
progbar.show_progress(prefix="Write", pos=0, total=total, display=display)
if rsp[0]:
old = 0
pos = 0
@ -515,7 +490,7 @@ class firehose(metaclass=LogBase):
self.cdc.write(wrdata)
self.show_progress("Write", total - bytestowrite, total, display)
progbar.show_progress(prefix="Write", pos=total - bytestowrite, total=total, display=display)
self.cdc.write(b'')
# time.sleep(0.2)
@ -552,12 +527,13 @@ class firehose(metaclass=LogBase):
pos = 0
bytestowrite = self.cfg.SECTOR_SIZE_IN_BYTES * num_partition_sectors
total = self.cfg.SECTOR_SIZE_IN_BYTES * num_partition_sectors
self.show_progress("Erase", 0, total, display)
progbar = progress(self.cfg.MaxPayloadSizeToTargetInBytes)
progbar.show_progress(prefix="Erase", pos=0, total=total, display=display)
if rsp[0]:
while bytestowrite > 0:
wlen = min(bytestowrite, self.cfg.MaxPayloadSizeToTargetInBytes)
self.cdc.write(empty[:wlen])
self.show_progress("Erase", total - bytestowrite, total, display)
progbar.show_progress(prefix="Erase", pos=total - bytestowrite, total=total, display=display)
bytestowrite -= wlen
pos += wlen
self.cdc.write(b'')
@ -579,11 +555,11 @@ class firehose(metaclass=LogBase):
def cmd_read(self, physical_partition_number, start_sector, num_partition_sectors, filename, display=True):
self.lasterror = b""
prog = 0
progbar = progress(self.cfg.SECTOR_SIZE_IN_BYTES)
if display:
self.info(
f"\nReading from physical partition {str(physical_partition_number)}, " +
f"sector {str(start_sector)}, sectors {str(num_partition_sectors)}")
print_progress(prog, 100, prefix='Progress:', suffix='Complete', bar_length=50)
with open(file=filename, mode="wb", buffering=self.cfg.MaxPayloadSizeFromTargetInBytes) as wr:
@ -602,15 +578,15 @@ class firehose(metaclass=LogBase):
return b""
bytestoread = self.cfg.SECTOR_SIZE_IN_BYTES * num_partition_sectors
total = bytestoread
show_progress = self.show_progress
show_progress = progbar.show_progress
usb_read = self.cdc.read
self.show_progress("Read", 0, total, display)
progbar.show_progress(prefix="Read", pos=0, total=total, display=display)
wMaxPacketSize = self.cdc.EP_IN.wMaxPacketSize
while bytestoread > 0:
data=usb_read(min(wMaxPacketSize, bytestoread))
data = usb_read(min(wMaxPacketSize, bytestoread))
wr.write(data)
bytestoread -= len(data)
show_progress("Read", total - bytestoread, total, display)
show_progress(prefix="Read", pos=total - bytestoread, total=total, display=display)
# time.sleep(0.2)
wd = self.wait_for_data()
info = self.xml.getlog(wd)
@ -644,6 +620,7 @@ class firehose(metaclass=LogBase):
f" physical_partition_number=\"{physical_partition_number}\"" + \
f" start_sector=\"{start_sector}\"/>\n</data>"
progbar = progress(self.cfg.SECTOR_SIZE_IN_BYTES)
rsp = self.xmlsend(data, self.skipresponse)
resData = bytearray()
if rsp[0]:
@ -655,13 +632,13 @@ class firehose(metaclass=LogBase):
bytestoread = self.cfg.SECTOR_SIZE_IN_BYTES * num_partition_sectors
total = bytestoread
if display:
print_progress(prog, 100, prefix='Progress:', suffix='Complete', bar_length=50)
progbar.show_progress(prefix="Read", pos=total - bytestoread, total=total, display=display)
while bytestoread > 0:
tmp = self.cdc.read(min(self.cdc.EP_IN.wMaxPacketSize, bytestoread))
size=len(tmp)
size = len(tmp)
bytestoread -= size
resData.extend(tmp)
self.show_progress("Read", total-bytestoread, total, display)
progbar.show_progress(prefix="Read", pos=total - bytestoread, total=total, display=display)
wd = self.wait_for_data()
info = self.xml.getlog(wd)
@ -681,7 +658,7 @@ class firehose(metaclass=LogBase):
if len(rsp) > 2 and not rsp[0]:
self.lasterror = rsp[2]
if display and prog != 100:
print_progress(100, 100, prefix='Progress:', suffix='Complete', bar_length=50)
progbar.show_progress(prefix="Read", pos=total, total=total, display=display)
return resData # Do not remove, needed for oneplus
def get_gpt(self, lun, gpt_num_part_entries, gpt_part_entry_size, gpt_part_entry_start_lba):
@ -805,9 +782,9 @@ class firehose(metaclass=LogBase):
self.info("Xiaomi EDL Auth detected.")
try:
self.modules = modules(fh=self, serial=self.serial,
supported_functions=self.supported_functions,
loglevel=self.__logger.level,
devicemodel=self.devicemodel, args=self.args)
supported_functions=self.supported_functions,
loglevel=self.__logger.level,
devicemodel=self.devicemodel, args=self.args)
except Exception as err: # pylint: disable=broad-except
self.modules = None
if self.modules.edlauth():
@ -876,23 +853,22 @@ class firehose(metaclass=LogBase):
self.info(f"MemoryName={self.cfg.MemoryName}")
self.info(f"Version={self.cfg.Version}")
if not self.cfg.SkipStorageInit and self.args["--memory"] is not None:
rsp = self.cmd_read_buffer(0, 1, 1, False)
if rsp == b"" and self.args["--memory"] is None:
if b"Failed to open the SDCC Device" in self.lasterror:
self.warning(
"Memory type eMMC doesn't seem to match (Failed to init). Trying to use UFS instead.")
self.cfg.MemoryName = "UFS"
return self.configure(0)
if b"ERROR: Failed to initialize (open whole lun) UFS Device slot" in self.lasterror:
self.warning(
"Memory type UFS doesn't seem to match (Failed to init). Trying to use eMMC instead.")
self.cfg.MemoryName = "eMMC"
return self.configure(0)
elif b"Attribute \'SECTOR_SIZE_IN_BYTES\'=4096 must be equal to disk sector size 512" in self.lasterror:
self.cfg.SECTOR_SIZE_IN_BYTES = 512
elif b"Attribute \'SECTOR_SIZE_IN_BYTES\'=512 must be equal to disk sector size 4096" in self.lasterror:
self.cfg.SECTOR_SIZE_IN_BYTES = 4096
rsp = self.cmd_read_buffer(0, 1, 1, False)
if rsp == b"" and self.args["--memory"] is None:
if b"Failed to open the SDCC Device" in self.lasterror:
self.warning(
"Memory type eMMC doesn't seem to match (Failed to init). Trying to use UFS instead.")
self.cfg.MemoryName = "UFS"
return self.configure(0)
if b"ERROR: Failed to initialize (open whole lun) UFS Device slot" in self.lasterror:
self.warning(
"Memory type UFS doesn't seem to match (Failed to init). Trying to use eMMC instead.")
self.cfg.MemoryName = "eMMC"
return self.configure(0)
elif b"Attribute \'SECTOR_SIZE_IN_BYTES\'=4096 must be equal to disk sector size 512" in self.lasterror:
self.cfg.SECTOR_SIZE_IN_BYTES = 512
elif b"Attribute \'SECTOR_SIZE_IN_BYTES\'=512 must be equal to disk sector size 4096" in self.lasterror:
self.cfg.SECTOR_SIZE_IN_BYTES = 4096
self.luns = self.getluns(self.args)
return True
@ -985,8 +961,8 @@ class firehose(metaclass=LogBase):
if self.serial is None or self.supported_functions is []:
try:
if os.path.exists("edl_config.json"):
pinfo=json.loads(open("edl_config.json","rb").read())
if self.supported_functions==[]:
pinfo = json.loads(open("edl_config.json", "rb").read())
if self.supported_functions == []:
if "supported_functions" in pinfo:
self.supported_functions = pinfo["supported_functions"]
if self.serial is None:
@ -1023,19 +999,19 @@ class firehose(metaclass=LogBase):
for cmd in [b"demacia", b"setprojmodel", b"setswprojmodel", b"setprocstart", b"SetNetType"]:
if cmd in data:
self.supported_functions.append(cmd.decode('utf-8'))
state={
"supported_functions" : self.supported_functions,
"programmer" : self.cfg.programmer,
"serial" : self.serial
state = {
"supported_functions": self.supported_functions,
"programmer": self.cfg.programmer,
"serial": self.serial
}
open("edl_config.json", "w").write(json.dumps(state))
except:
pass
#rsp = self.xmlsend(data, self.skipresponse)
# rsp = self.xmlsend(data, self.skipresponse)
if "getstorageinfo" in self.supported_functions and self.args["--memory"] is None:
storageinfo = self.cmd_getstorageinfo()
if storageinfo is not None and storageinfo!=[]:
if storageinfo is not None and storageinfo != []:
for info in storageinfo:
if "storage_info" in info:
try:
@ -1102,11 +1078,11 @@ class firehose(metaclass=LogBase):
return None
def cmd_test(self, cmd):
token="1234"
pk="1234"
data = "<?xml version=\"1.0\" ?>\n<data>\n<"+cmd+" token=\"" + token + "\" pk=\"" + pk + "\" />\n</data>"
token = "1234"
pk = "1234"
data = "<?xml version=\"1.0\" ?>\n<data>\n<" + cmd + " token=\"" + token + "\" pk=\"" + pk + "\" />\n</data>"
val = self.xmlsend(data)
if len(val)>1:
if len(val) > 1:
if b"raw hex token" in val[2]:
return True
if b"opcmd is not enabled" in val[2]:

25
edlclient/Library/firehose_client.py
View file

@ -14,6 +14,7 @@ from edlclient.Library.utils import do_tcp_server
from edlclient.Library.utils import LogBase, getint
from edlclient.Config.qualcomm_config import memory_type
from edlclient.Config.qualcomm_config import infotbl, msmids, secureboottbl, sochw
import fnmatch
try:
import xml.etree.cElementTree as ET
@ -297,11 +298,14 @@ class firehose_client(metaclass=LogBase):
if genxml:
guid_gpt.generate_rawprogram(lun, self.firehose.cfg.SECTOR_SIZE_IN_BYTES, storedir)
skipped = []
for skippart in skip:
filtered = fnmatch.filter(guid_gpt.partentries, skippart)
skipped.append(filtered)
for selpart in guid_gpt.partentries:
partition = guid_gpt.partentries[selpart]
partitionname = partition.name
if partition.name in skip:
if partition.name in skipped:
continue
filename = os.path.join(storedir, partitionname + ".bin")
self.info(
@ -917,23 +921,6 @@ class firehose_client(metaclass=LogBase):
self.info(
"[qfil] set partition({partition}) as bootable failed\n".format(partition=bootable))
elif cmd == "provision":
self.info("[ufs] provision...")
filename = options["<xmlfile>"]
if os.path.exists(filename):
fl = open(filename, "r")
for evt, elem in ET.iterparse(fl, events=["end"]):
if elem.tag == "ufs":
content = ElementTree.tostring(elem).decode("utf-8")
CMD = "<?xml version=\"1.0\" ?><data>\n {content} </data>".format(
content=content)
print(CMD)
self.firehose.xmlsend(CMD)
else:
self.error(f"File : {filename} not found.")
return False
self.info("[ufs] provision ok.")
else:
self.error("Unknown/Missing command, a command is required.")
return False

5
edlclient/Library/sparse.py
View file

@ -43,10 +43,7 @@ class QCSparse(metaclass=LogBase):
self.__logger.addHandler(fh)
def readheader(self):
buf = self.rf.read(0x1C)
if len(buf) != 28:
return False
header = unpack("<I4H4I", buf)
header = unpack("<I4H4I", self.rf.read(0x1C))
magic = header[0]
self.major_version = header[1]
self.minor_version = header[2]

71
edlclient/Library/utils.py
View file

@ -11,6 +11,8 @@ import shutil
import stat
import colorama
import copy
import datetime as dt
import time
from struct import unpack, pack
try:
@ -23,8 +25,77 @@ except ImportError:
print("Keystone library is missing (optional).")
class progress:
def __init__(self, pagesize):
self.progtime = 0
self.prog = 0
self.progpos = 0
self.start = time.time()
self.pagesize = pagesize
def calcProcessTime(self, starttime, cur_iter, max_iter):
telapsed = time.time() - starttime
if telapsed > 0 and cur_iter > 0:
testimated = (telapsed / cur_iter) * (max_iter)
finishtime = starttime + testimated
finishtime = dt.datetime.fromtimestamp(finishtime).strftime("%H:%M:%S") # in time
lefttime = testimated - telapsed # in seconds
return int(telapsed), int(lefttime), finishtime
else:
return 0, 0, ""
def show_progress(self, prefix, pos, total, display=True):
if pos != 0:
prog = round(float(pos) / float(total) * float(100), 1)
else:
prog = 0
if prog == 0:
self.prog = 0
self.start = time.time()
self.progtime = time.time()
self.progpos = pos
print_progress(prog, 100, prefix='Done',
suffix=prefix + ' (Sector 0x%X of 0x%X) %0.2f MB/s' %
(pos // self.pagesize,
total // self.pagesize,
0), bar_length=50)
if prog > self.prog:
if display:
t0 = time.time()
tdiff = t0 - self.progtime
datasize = (pos - self.progpos) / 1024 / 1024
if datasize != 0 and tdiff != 0:
try:
throughput = datasize / tdiff
except:
throughput = 0
else:
throughput = 0
telapsed, lefttime, finishtime = self.calcProcessTime(self.start, prog, 100)
hinfo = ""
if lefttime > 0:
sec = lefttime
if sec > 60:
min = sec // 60
sec = sec % 60
if min > 60:
h = min // 24
min = min % 24
hinfo = "%02dh:%02dm:%02ds left" % (h, min, sec)
else:
hinfo = "%02dm:%02ds left" % (min, sec)
else:
hinfo = "%02ds left" % sec
print_progress(prog, 100, prefix='Progress:',
suffix=prefix + f' (Sector 0x%X of 0x%X, {hinfo}) %0.2f MB/s' %
(pos // self.pagesize,
total // self.pagesize,
throughput), bar_length=50)
self.prog = prog
self.progpos = pos
self.progtime = t0
class structhelper:
pos = 0