morgan/python-bzz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Renamed decompressed_files/ to out/

Browse Source 
Fised a lot of logic
Set starting bit to 0x800, since that's after a big blank space in files, is the method bit we are expecting (0x0B) every time, and works to decomp all files (wow!)
This commit is contained in:
Morgan Kelley
2025-09-16 20:36:39 -06:00
parent 5fef2ec23d
commit f1c346a634
2 changed files with 95 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -1,6 +1,6 @@
# Project Specific # Project Specific
bin_extract/* bin_extract/*
decompressed_files/* out/*
bzz_mudule.py bzz_mudule.py
# Byte-compiled / optimized / DLL files # Byte-compiled / optimized / DLL files

178
main.py
View File

@@ -1,171 +1,166 @@
import os
from bitarray import bitarray from bitarray import bitarray
from pathlib import Path
class BZZCompressor: class BZZCompressor:
def decompress(self, input_file_path) -> bytes: def decompress(self, input_file_path, input_file, output_folder="out/") -> bytes:
data = bitarray(endian="big") data = bitarray(endian="big")
output_buffer = bitarray(endian="big") output_buffer = bytearray()
index = 0x800
# read the input file # read the input file
try: try:
with open(input_file_path, "rb") as input_file: with open(f"{input_file_path}/{input_file}", "rb") as input_file:
data.fromfile(input_file) data.fromfile(input_file)
data = data.tobytes()
except IOError: except IOError:
print("Could not open input file...") print("Could not open input file...")
raise raise
if len(data) > 9: if len(data) > 9:
# Getting our method, this is likely imprecise, since I'm one dealing with one method type, but it gets what I want # Getting our method, this is likely imprecise, since I'm one dealing with one method type, but it gets what I want
method = data[0:8] | bitarray("00001010") method = data[index]
# We move on to the next byte in data # We move on to the next byte in data
del data[0:8] index = index + 1
# Gathering variables based on the method according to https://problemkaputt.de/psxspx-cdrom-file-compression-bzz.htm # Gathering variables based on the method according to https://problemkaputt.de/psxspx-cdrom-file-compression-bzz.htm
# Note: bin(int)[2:].zfill(8) converts a number to an 8-bit binary string # Note: bin(int)[2:].zfill(8) converts a number to an 8-bit binary string
# `>> 3` is the same as dividing by 8 # `>> 3` is the same as dividing by 8
shifter = (method >> 3) & bitarray(bin(0x03)[2:].zfill(8)) shifter = (method >> 3) & 0x03
len_bits = (method & bitarray(bin(0x07)[2:].zfill(8))) ^ bitarray( len_bits = (method & 0x07) ^ 0x07
bin(0x07)[2:].zfill(8)
)
# The bin() function only returns the second half of the byte, so we pad the byte # The bin() function only returns the second half of the byte, so we pad the byte
len_mask = bitarray(bin((1 << int(len_bits.to01(), 2)) - 1)[2:].zfill(8)) len_mask = 1 << len_bits
threshold = len_mask >> 1 threshold = len_mask >> 1
if int(threshold.to01(), 2) > 0x07:
threshold = bitarray(bin(0x13).zfill(8)) if threshold > 0x07:
threshold = 0x13
len_table = [] len_table = []
for i in range(int(len_mask.to01(), 2)): for i in range(len_mask):
if i > int(threshold.to01(), 2): if i > threshold:
len_table.append( len_table.append((i - threshold << shifter) + threshold + 3)
(i - int(threshold.to01(), 2) << int(shifter.to01(), 2))
+ int(threshold.to01(), 2)
+ 3
)
else: else:
len_table.append(i + 3) len_table.append(i + 3)
num_flags = bitarray(bin(int(data[0:24].to01(), 2) + 1)[2:].zfill(24)) temp_flags = ""
del data[0:24]
print(f"Method: {method.tobytes()}") for item in data[index : index + 3]:
print(f"Shifter: {shifter.tobytes()}") temp_flags += bin(item)[2:].zfill(8)
print(f"Len Bits: {len_bits.tobytes()}")
print(f"Len Mask: {len_mask.tobytes()}") num_flags = int(temp_flags, 2) + 1
print(f"Threshold: {threshold.tobytes()}") index = index + 3
print(f"Method: {hex(method)}")
print(f"Shifter: {shifter}")
print(f"Len Bits: {bin(len_bits)}")
print(f"Len Mask: {bin(len_mask)}")
print(f"Threshold: {threshold}")
print(f"Len Table: {len_table}") print(f"Len Table: {len_table}")
print(f"Num Flags: {num_flags.tobytes()}") print(f"Num Flags: {bin(num_flags)}")
# Adding 0x100 here means the bitarray is a length of 9, and the first item is always 1 # Adding 0x100 here means the bitarray is a length of 9, and the first item is always 1
# This means that later, when we need to gather more flag bits, we aren't losing any data, or # This means that later, when we need to gather more flag bits, we aren't losing any data, or
# hitting an index out of bounds error # hitting an index out of bounds error
flag_bits = bitarray(bin(int(data[0:8].to01(), 2) + 0x100)[2:]) flag_bits = bitarray(bin(data[index] + 0x100)[2:])
del data[0:8] index = index + 1
print(f"Starting flag_bits: {flag_bits}") print(f"Starting flag_bits: {flag_bits}")
while int(num_flags.to01(), 2) > 0: while num_flags > 0:
carry = flag_bits[-1] carry = flag_bits[-1]
flag_bits = flag_bits >> 1 flag_bits = flag_bits >> 1
if len(flag_bits) > 8 and flag_bits[0] == 0:
flag_bits = flag_bits[1:]
print(f"Carry: {carry}") print(f"Carry: {carry}")
print(f"Flag Bits: {flag_bits}") print(f"Flag Bits: {flag_bits}")
# if we are down to only 0 bits, we are out of file-driven data # if we are down to only 0 bits, we are out of file-driven data
# Here we collect more flag bits and re-iterate the loop # Here we collect more flag bits and re-iterate the loop
if len(flag_bits.to01()) == 0: if int(flag_bits.to01(), 2) == 0x00:
flag_bits = bitarray(bin(int(data[0:8].to01(), 2) + 0x100)[2:]) flag_bits = bitarray(bin(data[index] + 0x100)[2:])
del data[0:8] index = index + 1
continue continue
# Carry means the next byte is raw data, no weird placement or indexing # Carry means the next byte is raw data, no weird placement or indexing
if carry: if carry:
if len(data) == 0: try:
print("Error processing file. Reached of data stream early.") output_buffer.append(data[index])
index = index + 1
except IndexError:
print(output_buffer)
print(
f"Error processing file. Reached of data stream early. Index: {index}"
)
return return
output_buffer.append(data[0:8])
del data[0:8]
# If Carry is 0, then we are doing actual decompression. This is the tricky part # If Carry is 0, then we are doing actual decompression. This is the tricky part
else: else:
# This shouldn't happen # This shouldn't happen
if len(data) <= 8: if len(data) <= index + 1:
print("Error processing file. Reached of data stream early.") print("Error processing file. Reached of data stream early.")
return return
# This is "temp" in our documentation # This is "temp" in our documentation
distance_data = data[0:16] temp = ""
del data[0:16] for item in data[index : index + 2]:
temp = temp + bin(item)[2:].zfill(8)
distance_data = int(temp, 2)
index = index + 2
# length here is the length of the data we are copying. # length here is the length of the data we are copying.
# We multiply by 8 since we are working with bits instead of bytes # We multiply by 8 since we are working with bits instead of bytes
length = ( length = len_table[(distance_data & len_mask) - 1]
len_table[
int(
(
distance_data
& bitarray(
bin(int(len_mask.to01(), 2))[2:].zfill(16)
)
).to01(),
2,
)
]
* 8
)
# Displacement is how far back in the existing output_buffer we are # Displacement is how far back in the existing output_buffer we are
# looking to copy from. We multiply by 8 since we are working with bits and not bytes # looking to copy from. We multiply by 8 since we are working with bits and not bytes
displacement = ( displacement = distance_data >> len_bits
int((distance_data >> int(len_bits.to01(), 2)).to01(), 2) * 8
)
# This shouldn't happen # This shouldn't happen
if displacement <= 0: if displacement <= 0:
print( print(
"Error processing file. Displacement was less than or equal to 0" f"Error processing file. Displacement was less than or equal to 0.\n"
+ f"Distance Data: {distance_data}. Displacement: {displacement}. Index: {hex(index)}"
) )
return return
print(f"Output Buffer Size (in bits): {len(output_buffer)}") print(f"Output Buffer Size {len(output_buffer)}")
print(f"Distance Data: {distance_data.tobytes()}") print(f"Distance Data: {distance_data}")
print(f"Displacement (in bits): {displacement}") print(f"Displacement: {displacement}")
print(f"Length (in bits): {length}") print(f"Length: {length}")
# Here we copy bit by bit from earlier in the output buffer. # Here we copy bit by bit from earlier in the output buffer.
# we use this instead of index slicing since the slice could lead to # we use this instead of index slicing since the slice could lead to
# data we are currently copying into the buffer # data we are currently copying into the buffer
start_index = len(output_buffer) - displacement copy_index = len(output_buffer) - displacement
# If start index is less than 0, we'll be checking something like output_buffer[-2] # If start index is less than 0, we'll be checking something like output_buffer[-2]
# or smth, which will have an IndexOutOfBounds exception # or smth, which will have an IndexOutOfBounds exception
if start_index < 0: if copy_index < 0:
print(output_buffer)
print("Error decompressing file. Start Index was out of range.") print("Error decompressing file. Start Index was out of range.")
return return
for i in range(length): for i in range(length):
output_buffer.append(output_buffer[start_index + i]) output_buffer.append(output_buffer[copy_index + i])
num_flags = bitarray(bin(int(num_flags.to01(), 2) - 1)[2:].zfill(24)) num_flags = num_flags - 1
if len(data) > 0: if len(data) > index:
output_buffer.append( for item in data[index:]:
bitarray(data.to01() + "0".join("" for i in range(8 - len(data)))) output_buffer.append(item)
)
else: else:
# If the file is less than 9 bits, it's just output # If the file is less than 9 bits, it's just output
for i in data: for i in data:
output_buffer.append(i) output_buffer.append(i)
out_data = b"".join(output_buffer.tobytes()) # This handoff is so I can change buffer logic without breaking write-out logic
out_data = output_buffer
try: try:
if "bin_extract" in input_file_path[0:11]: if "bin_extract" in input_file_path[0:11]:
@@ -174,20 +169,35 @@ class BZZCompressor:
output_file_path = input_file_path output_file_path = input_file_path
if "/" in output_file_path: if "/" in output_file_path:
output_file_name = output_file_path.split("/")[-1] output_file_name = output_file_path.split("/")[-1] + ".file"
else: else:
output_file_name = output_file_path output_file_name = output_file_path + ".file"
# TODO: Create file path, if it doesn't exist # TODO: Create file path, if it doesn't exist
with open( with open(f"{output_folder}/{output_file_name}", "wb") as outfile:
f"decompressed_files/{output_file_name[:4]}.file", "wb"
) as outfile:
outfile.write(out_data) outfile.write(out_data)
print(f"File {output_file_name} saved successfully!") print(f"File {output_file_name} saved successfully!")
except IOError: except IOError as e:
print(f"Unable to write file for {input_file_path}") print(
f"Unable to write file for {input_file_path}/{input_file}. Error: {e}"
)
compressor = BZZCompressor() if __name__ == "__main__":
compressor = BZZCompressor()
compressor.decompress("bin_extract/level/mc/mccave01.bzz") for dirpath, dirnames, filenames in os.walk("./bin_extract"):
print(f"{dirpath} | {', '.join(filenames)}")
for file in filenames:
if file[-4:] == ".bzz":
output_folder_path = Path(f"out/{'/'.join(dirpath.split("/")[2:])}")
output_folder_path.mkdir(parents=True, exist_ok=True)
try:
compressor.decompress(dirpath, file, output_folder_path)
except Exception as e:
print(
f"Error while decompressing {output_folder_path}/{file}. Error: {e}"
)
continue