Skip to content

Commit

Permalink
Generate extra xbox files (#198)
Browse files Browse the repository at this point in the history
* Generate extra xbox files

* Remove unnecessary utils.xbox reference

* indent range printing

* change ss range info header

* Cleanup, only write to DMI/PFI/SS if they don't exist

* Remove unnecessary line

* Only generate extra xbox files during dump phase

* Create dvd_split

* clang formatting

* Fix includes

* Fix build

* Use constant

* Fix dvd split

* Cleanup, remove option
  • Loading branch information
Deterous authored Jan 21, 2025
1 parent fc82107 commit 63a9eab
Show file tree
Hide file tree
Showing 7 changed files with 229 additions and 56 deletions.
3 changes: 2 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -120,14 +120,15 @@ target_sources(redumper
"cd/offset_manager.ixx"
"cd/protection.ixx"
"cd/scrambler.ixx"
"cd/split.ixx"
"cd/cd_split.ixx"
"cd/subcode.ixx"
"cd/toc.ixx"
"crc/crc.ixx"
"crc/crc16_gsm.ixx"
"crc/crc32.ixx"
"dvd/dvd_dump.ixx"
"dvd/dvd_key.ixx"
"dvd/dvd_split.ixx"
"dvd/css/css.ixx"
"filesystem/iso9660/iso9660.ixx"
"filesystem/iso9660/iso9660_browser.ixx"
Expand Down
2 changes: 1 addition & 1 deletion cd/split.ixx → cd/cd_split.ixx
Original file line number Diff line number Diff line change
Expand Up @@ -862,7 +862,7 @@ void disc_offset_normalize_records(std::vector<SyncAnalyzer::Record> &records, s
}


export void redumper_split(Context &ctx, Options &options)
export void redumper_split_cd(Context &ctx, Options &options)
{
image_check_empty(options);

Expand Down
71 changes: 23 additions & 48 deletions dvd/dvd_dump.ixx
Original file line number Diff line number Diff line change
Expand Up @@ -476,19 +476,17 @@ export bool redumper_dump_dvd(Context &ctx, const Options &options, DumpMode dum

if(dump_mode == DumpMode::REFINE && !options.force_refine)
{
// if not dumping, compare security sector to stored to make sure it's the same disc
if(!std::filesystem::exists(security_sector_fn))
{
throw_line("disc / file security sector doesn't match, refining from a different disc?");
}
else
// if not dumping, compare cleaned security sector to stored to make sure it's the same disc
bool invalid_ss = true;
if(std::filesystem::exists(security_sector_fn))
{
auto refined_security_sector = read_vector(security_sector_fn);
clean_xbox_security_sector(refined_security_sector);

if(refined_security_sector != security_sector)
throw_line("disc / file security sector doesn't match, refining from a different disc?");
invalid_ss = (refined_security_sector != security_sector);
}

if(invalid_ss)
throw_line("disc / file security sector doesn't match, refining from a different disc?");
}

auto &structure = physical_structures.front();
Expand All @@ -507,39 +505,15 @@ export bool redumper_dump_dvd(Context &ctx, const Options &options, DumpMode dum
auto &ss_layer_descriptor = (READ_DVD_STRUCTURE_LayerDescriptor &)security_sector[0];

int32_t ss_lba_first = sign_extend<24>(endian_swap(ss_layer_descriptor.data_start_sector));
int32_t ss_layer0_last = sign_extend<24>(endian_swap(ss_layer_descriptor.layer0_end_sector));

uint32_t l1_padding_length = ss_lba_first - layer0_last - 1;
if(xgd_type == XGD_Type::XGD3)
l1_padding_length += 4096;

// extract security sector ranges
bool is_xgd1 = (xgd_type == XGD_Type::XGD1);

const auto media_specific_offset = offsetof(READ_DVD_STRUCTURE_LayerDescriptor, media_specific);
uint8_t num_ss_regions = ss_layer_descriptor.media_specific[1632 - media_specific_offset];
// partial pre-compute of conversion to Layer 1
const uint32_t layer1_offset = (ss_layer0_last * 2) - 0x30000 + 1;

for(int ss_pos = 1633 - media_specific_offset, i = 0; i < num_ss_regions; ss_pos += 9, ++i)
{
uint32_t start_psn = ((uint32_t)ss_layer_descriptor.media_specific[ss_pos + 3] << 16) | ((uint32_t)ss_layer_descriptor.media_specific[ss_pos + 4] << 8)
| (uint32_t)ss_layer_descriptor.media_specific[ss_pos + 5];
uint32_t end_psn = ((uint32_t)ss_layer_descriptor.media_specific[ss_pos + 6] << 16) | ((uint32_t)ss_layer_descriptor.media_specific[ss_pos + 7] << 8)
| (uint32_t)ss_layer_descriptor.media_specific[ss_pos + 8];
if((i < 8 && is_xgd1) || (i == 0 && !is_xgd1))
{
// Layer 0
xbox_skip_ranges.push_back({ start_psn - 0x30000, end_psn - 0x30000 });
}
else if((i < 16 && is_xgd1) || (i == 3 && !is_xgd1))
{
// Layer 1
xbox_skip_ranges.push_back({ layer1_offset - (start_psn ^ 0xFFFFFF), layer1_offset - (end_psn ^ 0xFFFFFF) });
}
}
// extract security sector ranges from security sector
xbox_skip_ranges = get_security_sector_ranges(ss_layer_descriptor);

// append L1 padding to ranges
// append L1 padding to skip ranges
xbox_skip_ranges.push_back({ sectors_count, sectors_count + l1_padding_length - 1 });

// sort the skip ranges
Expand Down Expand Up @@ -738,13 +712,13 @@ export bool redumper_dump_dvd(Context &ctx, const Options &options, DumpMode dum
uint32_t refine_counter = 0;
uint32_t refine_retries = options.retries ? options.retries : 1;

uint32_t errors_scsi = 0;
Errors errors = {};
// FIXME: verify memory usage for largest bluray and chunk it if needed
if(dump_mode != DumpMode::DUMP)
{
std::vector<State> state_buffer(sectors_count);
read_entry(fs_state, (uint8_t *)state_buffer.data(), sizeof(State), 0, sectors_count, 0, (uint8_t)State::ERROR_SKIP);
errors_scsi = std::count(state_buffer.begin(), state_buffer.end(), State::ERROR_SKIP);
errors.scsi = std::count(state_buffer.begin(), state_buffer.end(), State::ERROR_SKIP);
}

ROMEntry rom_entry(iso_path.filename().string());
Expand Down Expand Up @@ -784,7 +758,7 @@ export bool redumper_dump_dvd(Context &ctx, const Options &options, DumpMode dum
{
// skip at most to the end of the security sector range
sectors_to_read = std::min(sectors_to_read, xbox_skip_ranges[skip_range_idx].second + 1 - s);
progress_output(s, sectors_count, errors_scsi);
progress_output(s, sectors_count, errors.scsi);

std::vector<uint8_t> zeroes(sectors_to_read * FORM1_DATA_SIZE);
write_entry(fs_iso, zeroes.data(), FORM1_DATA_SIZE, s, sectors_to_read, 0);
Expand Down Expand Up @@ -837,7 +811,7 @@ export bool redumper_dump_dvd(Context &ctx, const Options &options, DumpMode dum

if(read)
{
progress_output(s, sectors_count, errors_scsi);
progress_output(s, sectors_count, errors.scsi);

std::vector<uint8_t> drive_data(sectors_at_once * FORM1_DATA_SIZE);

Expand All @@ -859,7 +833,7 @@ export bool redumper_dump_dvd(Context &ctx, const Options &options, DumpMode dum
}

if(dump_mode == DumpMode::DUMP)
errors_scsi += sectors_to_read;
errors.scsi += sectors_to_read;
else if(dump_mode == DumpMode::REFINE)
{
++refine_counter;
Expand Down Expand Up @@ -898,7 +872,7 @@ export bool redumper_dump_dvd(Context &ctx, const Options &options, DumpMode dum
std::copy(drive_data.begin() + i * FORM1_DATA_SIZE, drive_data.begin() + (i + 1) * FORM1_DATA_SIZE, file_data.begin() + i * FORM1_DATA_SIZE);
file_state[i] = State::SUCCESS;

--errors_scsi;
--errors.scsi;
}

refine_counter = 0;
Expand All @@ -923,7 +897,7 @@ export bool redumper_dump_dvd(Context &ctx, const Options &options, DumpMode dum
file_state[i] = State::ERROR_SKIP;
update = true;

++errors_scsi;
++errors.scsi;
}
}

Expand All @@ -933,7 +907,7 @@ export bool redumper_dump_dvd(Context &ctx, const Options &options, DumpMode dum
}
}

if(dump_mode == DumpMode::DUMP && !errors_scsi)
if(dump_mode == DumpMode::DUMP && !errors.scsi)
rom_entry.update(file_data.data(), sectors_to_read * FORM1_DATA_SIZE);

if(signal.interrupt())
Expand All @@ -956,21 +930,22 @@ export bool redumper_dump_dvd(Context &ctx, const Options &options, DumpMode dum

if(!signal.interrupt())
{
progress_output(sectors_count, sectors_count, errors_scsi);
progress_output(sectors_count, sectors_count, errors.scsi);
LOG("");
}
LOG("");

LOG("media errors: ");
LOG(" SCSI: {}", errors_scsi);
LOG(" SCSI: {}", errors.scsi);
ctx.dump_errors = errors;

if(signal.interrupt())
signal.raiseDefault();

if(dump_mode == DumpMode::DUMP && !errors_scsi)
if(dump_mode == DumpMode::DUMP && !errors.scsi)
ctx.dat = std::vector<std::string>(1, rom_entry.xmlLine());

return errors_scsi;
return errors.scsi;
}

}
145 changes: 145 additions & 0 deletions dvd/dvd_split.ixx
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
module;
#include <cstdint>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <string>
#include <vector>
#include "throw_line.hh"

export module dvd.split;

import dump;
import options;
import rom_entry;
import scsi.mmc;
import utils.file_io;
import utils.logger;
import utils.misc;
import utils.xbox;



namespace gpsxre
{

const uint32_t DVD_DESCRIPTOR_SIZE = 2048;


void generate_extra_xbox(Context &ctx, Options &options)
{
image_check_empty(options);

auto image_prefix = (std::filesystem::path(options.image_path) / options.image_name).string();

// do not attempt to generate .ss, .dmi or .pfi for non-xbox discs (dumps without .security)
std::filesystem::path security_path(image_prefix + ".security");
if(std::filesystem::exists(security_path))
{
// trim the 4 byte header from .manufacturer and write it to a .dmi (if it doesn't exist)
std::filesystem::path manufacturer_path(image_prefix + ".manufacturer");
std::filesystem::path dmi_path(image_prefix + ".dmi");
if(std::filesystem::exists(manufacturer_path))
{
if(std::filesystem::exists(dmi_path) && !options.overwrite)
{
LOG("warning: file already exists ({}.dmi)", options.image_name);
}
else
{
auto manufacturer = read_vector(manufacturer_path);
if(!manufacturer.empty() && manufacturer.size() == DVD_DESCRIPTOR_SIZE + 4)
{
manufacturer.erase(manufacturer.begin(), manufacturer.begin() + 4);
write_vector(dmi_path, manufacturer);

ROMEntry dmi_rom_entry(dmi_path.filename().string());
dmi_rom_entry.update(manufacturer.data(), DVD_DESCRIPTOR_SIZE);
if(!ctx.dat.has_value())
ctx.dat = std::vector<std::string>();
ctx.dat->push_back(dmi_rom_entry.xmlLine());
}
else
{
LOG("warning: could not generate DMI, unexpected file size ({})", manufacturer_path.filename().string());
}
}
}

// trim the 4 byte header from .physical and write it to a .pfi (if it doesn't exist)
std::filesystem::path physical_path(image_prefix + ".physical");
std::filesystem::path pfi_path(image_prefix + ".pfi");
if(std::filesystem::exists(physical_path))
{
if(std::filesystem::exists(pfi_path) && !options.overwrite)
{
LOG("warning: file already exists ({}.pfi)", options.image_name);
}
else
{
auto physical = read_vector(physical_path);
if(!physical.empty() && physical.size() == DVD_DESCRIPTOR_SIZE + 4)
{
physical.erase(physical.begin(), physical.begin() + 4);
write_vector(pfi_path, physical);

ROMEntry pfi_rom_entry(pfi_path.filename().string());
pfi_rom_entry.update(physical.data(), DVD_DESCRIPTOR_SIZE);
if(!ctx.dat.has_value())
ctx.dat = std::vector<std::string>();
ctx.dat->push_back(pfi_rom_entry.xmlLine());
}
else
{
LOG("warning: could not generate PFI, unexpected file size ({})", physical_path.filename().string());
}
}
}

// clean the .security and write it to a .ss (if it doesn't exist)
std::filesystem::path ss_path(image_prefix + ".ss");
if(std::filesystem::exists(ss_path) && !options.overwrite)
{
LOG("warning: file already exists ({}.ss)", options.image_name);
}
else
{
auto security = read_vector(security_path);
if(!security.empty() && security.size() == DVD_DESCRIPTOR_SIZE)
{
clean_xbox_security_sector(security);
write_vector(ss_path, security);

ROMEntry ss_rom_entry(ss_path.filename().string());
ss_rom_entry.update(security.data(), DVD_DESCRIPTOR_SIZE);
if(!ctx.dat.has_value())
ctx.dat = std::vector<std::string>();
ctx.dat->push_back(ss_rom_entry.xmlLine());

LOG("security sector ranges:");
auto security_ranges = get_security_sector_ranges((READ_DVD_STRUCTURE_LayerDescriptor &)security[0]);
for(const auto &range : security_ranges)
{
LOG(" {}-{}", range.first, range.second);
}
}
else
{
LOG("warning: could not generate SS, unexpected file size ({})", security_path.filename().string());
}
}
}
}


export void redumper_split_dvd(Context &ctx, Options &options)
{
// generate .dmi, .pfi, .ss if xbox disc
generate_extra_xbox(ctx, options);

// prevent hash generation for ISO with scsi errors
if(ctx.dump_errors && ctx.dump_errors->scsi && !options.force_split)
throw_line("{} scsi errors detected, unable to continue", ctx.dump_errors->scsi);
}

}
8 changes: 8 additions & 0 deletions hash.ixx
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,14 @@ export void redumper_hash(Context &ctx, Options &options)
else if(std::filesystem::exists(image_prefix + ".iso"))
{
files.push_back(image_prefix + ".iso");

// hash xbox extras
if(std::filesystem::exists(image_prefix + ".dmi"))
files.push_back(image_prefix + ".dmi");
if(std::filesystem::exists(image_prefix + ".pfi"))
files.push_back(image_prefix + ".pfi");
if(std::filesystem::exists(image_prefix + ".ss"))
files.push_back(image_prefix + ".ss");
}
else
throw_line("image file not found");
Expand Down
22 changes: 16 additions & 6 deletions redumper.ixx
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import drive;
import dump;
import dvd.dump;
import dvd.key;
import dvd.split;
import hash;
import info;
import options;
Expand Down Expand Up @@ -151,6 +152,15 @@ void redumper_eject(Context &ctx, Options &options)
}


void redumper_split(Context &ctx, Options &options)
{
if(profile_is_cd(ctx.current_profile))
redumper_split_cd(ctx, options);
else
redumper_split_dvd(ctx, options);
}


const std::map<std::string, std::pair<bool, void (*)(Context &, Options &)>> COMMAND_HANDLERS{
// COMMAND DRIVE HANDLER
{ "rings", { true, redumper_rings } },
Expand Down Expand Up @@ -222,14 +232,14 @@ std::list<std::string> get_cd_batch_commands(Context &ctx, const std::string &co
: eject ? std::list<std::string>{ "dump", "protection", "refine", "eject", "split", "hash", "info" }
: std::list<std::string>{ "dump", "protection", "refine", "split", "hash", "info" };
else if(profile_is_dvd(ctx.current_profile))
return eject ? std::list<std::string>{ "dump", "refine", "dvdkey", "eject", "hash", "info" }
: std::list<std::string>{ "dump", "refine", "dvdkey", "hash", "info" };
return eject ? std::list<std::string>{ "dump", "refine", "dvdkey", "eject", "split", "hash", "info" }
: std::list<std::string>{ "dump", "refine", "dvdkey", "split", "hash", "info" };
else if(profile_is_bluray(ctx.current_profile))
return eject ? std::list<std::string>{ "dump", "refine", "eject", "hash", "info" }
: std::list<std::string>{ "dump", "refine", "hash", "info" };
return eject ? std::list<std::string>{ "dump", "refine", "eject", "split", "hash", "info" }
: std::list<std::string>{ "dump", "refine", "split", "hash", "info" };
else if(profile_is_hddvd(ctx.current_profile))
return eject ? std::list<std::string>{ "dump", "refine", "eject", "hash", "info" }
: std::list<std::string>{ "dump", "refine", "hash", "info" };
return eject ? std::list<std::string>{ "dump", "refine", "eject", "split", "hash", "info" }
: std::list<std::string>{ "dump", "refine", "split", "hash", "info" };
else
return std::list<std::string>{};
// clang-format on
Expand Down
Loading

0 comments on commit 63a9eab

Please sign in to comment.