morgan/python-bzz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Started parsing header data

Browse Source
This commit is contained in:
Morgan Kelley
2025-09-18 22:00:37 -06:00
parent eb0327f753
commit c24cca2373
1 changed files with 163 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

279
main.py
View File

@@ -5,159 +5,200 @@ from pathlib import Path
class BZZCompressor: class BZZCompressor:
def decompress(self, input_file_path, input_file, output_folder="out/") -> bytes: def decompress(self, input_file_path, input_file, output_folder="out/") -> bytes:
data = bitarray(endian="big") data = bytes()
output_buffer = bytearray() output_buffer = bytearray()
index = 0x800 overflow_buffer = bytearray()
# read the input file # read the input file
try: try:
with open(f"{input_file_path}/{input_file}", "rb") as infile: with open(f"{input_file_path}/{input_file}", "rb") as infile:
data.fromfile(infile) temp = bitarray(endian="big")
data = data.tobytes() temp.fromfile(infile)
data = temp.tobytes()
except IOError: except IOError:
print("Could not open input file...") print("Could not open input file...")
raise raise
if len(data) > 9: ##############################################################################
# Getting our method, this is likely imprecise, since I'm one dealing with one method type, but it gets what I want #
method = data[index] # Reading the Headers from the file.
# We move on to the next byte in data #
index = index + 1 # This includes the version, some garbage 0s, the number of files, and the
# file list (probably), and finally a Checksum
#
##############################################################################
# Gathering variables based on the method according to https://problemkaputt.de/psxspx-cdrom-file-compression-bzz.htm # This is always 1, 0, 0, 0. so I'm just having fun
# Note: bin(int)[2:].zfill(8) converts a number to an 8-bit binary string bzz_version = f"v{data[0]}.{data[1]}.{data[2]}.{data[3]}"
game_id = "".join([str(int(bin(item)[2:], 2)) for item in data[4:8]])
# `>> 3` is the same as dividing by 8 num_files = 0
shifter = (method >> 3) & 0x03
len_bits = (method & 0x07) ^ 0x07
# The bin() function only returns the second half of the byte, so we pad the byte for item in data[8:12]:
len_mask = 1 << len_bits num_files += int(bin(item)[2:].zfill(8), 2)
threshold = len_mask >> 1 print(f"BZZ Version: {bzz_version}")
print(f"Game ID: {game_id}")
print(f"Number of Files: {num_files}")
if threshold > 0x07: files = []
threshold = 0x13
len_table = [] for i in range(num_files):
tmp = (i) * 12
files.append(
{
"pt_a": data[12 + tmp : 12 + tmp + 4],
"pt_b": data[12 + tmp + 4 : 12 + tmp + 8],
"pt_c": data[12 + tmp + 8 : 12 + tmp + 12],
}
)
for i in range(len_mask): for i, file in enumerate(files):
if i > threshold: print(f"File {i+1}'s Data: {file}")
len_table.append((i - threshold << shifter) + threshold + 3)
else:
len_table.append(i + 3)
temp_flags = "" checksum = data[0x7FC:0x800]
print(f"Checksum: {checksum}")
for item in data[index : index + 3]: ##############################################################################
temp_flags += bin(item)[2:].zfill(8) #
# This is the File Loop, where we process
# individual files from the .bzz
#
##############################################################################
index = 0x800
num_flags = int(temp_flags, 2) + 1 # Getting our method, this is likely imprecise, since I'm one dealing with one
index = index + 3 # method type, but it gets what I want
method = data[index]
# We move on to the next byte in data
index = index + 1
print(f"Method: {hex(method)}") # Gathering variables based on the method according to
print(f"Shifter: {shifter}") # https://problemkaputt.de/psxspx-cdrom-file-compression-bzz.htm
print(f"Len Bits: {bin(len_bits)}") # Note: bin(int)[2:].zfill(8) converts a number to an 8-bit binary string
print(f"Len Mask: {bin(len_mask)}")
print(f"Threshold: {threshold}")
print(f"Len Table: {len_table}")
print(f"Num Flags: {bin(num_flags)}")
# Adding 0x100 here means the bitarray is a length of 9, and the first item is always 1 # `>> 3` is the same as dividing by 8
# This means that later, when we need to gather more flag bits, we aren't losing any data, or shifter = (method >> 3) & 0x03
# hitting an index out of bounds error len_bits = (method & 0x07) ^ 0x07
flag_bits = bitarray(bin(data[index] + 0x100)[2:])
index = index + 1
print(f"Starting flag_bits: {flag_bits}") # The bin() function only returns the second half of the byte, so we pad the byte
len_mask = 1 << len_bits
while num_flags > 0: threshold = len_mask >> 1
carry = flag_bits[-1]
flag_bits = flag_bits >> 1
print(f"Carry: {carry}") if threshold > 0x07:
print(f"Flag Bits: {flag_bits}") threshold = 0x13
# if we are down to only 0 bits, we are out of file-driven data len_table = []
# Here we collect more flag bits and re-iterate the loop
if int(flag_bits.to01(), 2) == 0x00: for i in range(len_mask):
flag_bits = bitarray(bin(data[index] + 0x100)[2:]) if i > threshold:
len_table.append((i - threshold << shifter) + threshold + 3)
else:
len_table.append(i + 3)
temp_flags = ""
for item in data[index : index + 3]:
temp_flags += bin(item)[2:].zfill(8)
num_flags = int(temp_flags, 2) + 1
index = index + 3
print(f"Method: {hex(method)}")
print(f"Shifter: {shifter}")
print(f"Len Bits: {bin(len_bits)}")
print(f"Len Mask: {bin(len_mask)}")
print(f"Threshold: {threshold}")
print(f"Len Table: {len_table}")
print(f"Loops (based on num flags): {num_flags}")
# Adding 0x100 here means the bitarray is a length of 9, and the first item is always 1
# This means that later, when we need to gather more flag bits, we aren't losing any data, or
# hitting an index out of bounds error
flag_bits = bitarray(bin(data[index] + 0x100)[2:])
index = index + 1
while num_flags > 0:
carry = flag_bits[-1]
flag_bits = flag_bits >> 1
# if we are down to only 0 bits, we are out of file-driven data
# Here we collect more flag bits and re-iterate the loop
if int(flag_bits.to01(), 2) == 0x00:
flag_bits = bitarray(bin(data[index] + 0x100)[2:])
index = index + 1
continue
# Carry means the next byte is raw data, no weird placement or indexing
if carry:
try:
output_buffer.append(data[index])
index = index + 1 index = index + 1
continue except IndexError:
print(output_buffer)
print(
f"Error processing file. Reached of data stream early. Index: {index}"
)
return
# Carry means the next byte is raw data, no weird placement or indexing # If Carry is 0, then we are doing actual decompression. This is the tricky part
if carry: else:
try: # This shouldn't happen
output_buffer.append(data[index]) if len(data) <= index + 1:
index = index + 1 print("Error processing file. Reached of data stream early.")
except IndexError: return
print(output_buffer)
print(
f"Error processing file. Reached of data stream early. Index: {index}"
)
return
# If Carry is 0, then we are doing actual decompression. This is the tricky part # This is "temp" in our documentation
else: temp = ""
# This shouldn't happen for item in data[index : index + 2]:
if len(data) <= index + 1: temp = temp + bin(item)[2:].zfill(8)
print("Error processing file. Reached of data stream early.")
return
# This is "temp" in our documentation distance_data = int(temp, 2)
temp = "" index = index + 2
for item in data[index : index + 2]:
temp = temp + bin(item)[2:].zfill(8)
distance_data = int(temp, 2) # length here is the length of the data we are copying.
index = index + 2 # We multiply by 8 since we are working with bits instead of bytes
length = len_table[(distance_data & len_mask) - 1]
# length here is the length of the data we are copying. # Displacement is how far back in the existing output_buffer we are
# We multiply by 8 since we are working with bits instead of bytes # looking to copy from. We multiply by 8 since we are working with bits and not bytes
length = len_table[(distance_data & len_mask) - 1] displacement = distance_data >> len_bits
# Displacement is how far back in the existing output_buffer we are # This shouldn't happen
# looking to copy from. We multiply by 8 since we are working with bits and not bytes if displacement <= 0:
displacement = distance_data >> len_bits print(
f"Error processing file. Displacement was less than or equal to 0.\n"
+ f"Distance Data: {distance_data}. Displacement: {displacement}. Index: {hex(index)}"
)
return
# This shouldn't happen # print(f"Output Buffer Size {len(output_buffer)}")
if displacement <= 0: # print(f"Distance Data: {distance_data}")
print( # print(f"Displacement: {displacement}")
f"Error processing file. Displacement was less than or equal to 0.\n" # print(f"Length: {length}")
+ f"Distance Data: {distance_data}. Displacement: {displacement}. Index: {hex(index)}"
)
return
print(f"Output Buffer Size {len(output_buffer)}") # Here we copy bit by bit from earlier in the output buffer.
print(f"Distance Data: {distance_data}") # we use this instead of index slicing since the slice could lead to
print(f"Displacement: {displacement}") # data we are currently copying into the buffer
print(f"Length: {length}") copy_index = len(output_buffer) - displacement
# Here we copy bit by bit from earlier in the output buffer. # If start index is less than 0, we'll be checking something like output_buffer[-2]
# we use this instead of index slicing since the slice could lead to # or smth, which will have an IndexOutOfBounds exception
# data we are currently copying into the buffer if copy_index < 0:
copy_index = len(output_buffer) - displacement print(output_buffer)
print("Error decompressing file. Start Index was out of range.")
return
# If start index is less than 0, we'll be checking something like output_buffer[-2] for i in range(length):
# or smth, which will have an IndexOutOfBounds exception output_buffer.append(output_buffer[copy_index + i])
if copy_index < 0:
print(output_buffer)
print("Error decompressing file. Start Index was out of range.")
return
for i in range(length): num_flags = num_flags - 1
output_buffer.append(output_buffer[copy_index + i])
num_flags = num_flags - 1 if len(data) > index:
for item in data[index:]:
if len(data) > index: overflow_buffer.append(item)
for item in data[index:]:
output_buffer.append(item)
else:
# If the file is less than 9 bits, it's just output
for i in data:
output_buffer.append(i)
# This handoff is so I can change buffer logic without breaking write-out logic # This handoff is so I can change buffer logic without breaking write-out logic
out_data = output_buffer out_data = output_buffer
@@ -166,6 +207,12 @@ class BZZCompressor:
with open(f"{output_folder}/{input_file}.file", "wb") as outfile: with open(f"{output_folder}/{input_file}.file", "wb") as outfile:
outfile.write(out_data) outfile.write(out_data)
print(f"File {output_folder}/{input_file}.file saved successfully!") print(f"File {output_folder}/{input_file}.file saved successfully!")
with open(f"{output_folder}/{input_file}.overflow.file", "wb") as outfile:
outfile.write(overflow_buffer)
print(
f"File {output_folder}/{input_file}.overflow.file saved successfully!"
)
except IOError as e: except IOError as e:
print( print(
f"Unable to write file for {input_file_path}/{input_file}. Error: {e}" f"Unable to write file for {input_file_path}/{input_file}. Error: {e}"
@@ -179,7 +226,7 @@ if __name__ == "__main__":
print(f"{dirpath} | {', '.join(filenames)}") print(f"{dirpath} | {', '.join(filenames)}")
for file in filenames: for file in filenames:
if file[-4:] == ".bzz": if file[-4:] == ".bzz" and file == "language.bzz":
output_folder_path = Path(f"out/{'/'.join(dirpath.split("/")[2:])}") output_folder_path = Path(f"out/{'/'.join(dirpath.split("/")[2:])}")
output_folder_path.mkdir(parents=True, exist_ok=True) output_folder_path.mkdir(parents=True, exist_ok=True)