Itms/0ad
1
0
Fork 0
forked from 0ad/0ad
Code Pull Requests Activity

Mostly complete archive-builder

Browse Source 
This was SVN commit r1599.
This commit is contained in:
Ykkrosh 2004-12-30 17:42:22 +00:00
parent 7efc7a7903
commit 170c511c58
1 changed files with 93 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
source/tools/archbuilder/archbuild.pl
View File

@ -10,7 +10,7 @@ It currently handles only files inside mods/official, since all other data-sets
Basic overview:
* Get a list of all files that the game uses from a particular mod.
* Get a list of all files that the game can use from a particular mod.
* Clean up the list by removing unnecessary files.
* Sort them, using the file-lists that the game generates.
* Create the new archive.
@ -18,46 +18,48 @@ Basic overview:
More detailed overview:
* Get a list of all files that the game uses:
Files have:
* Get a list of files:
Check the mods/<modname> directory recursively for files. Files are added in alphabetic order. When a zip file is encountered in the root directory, all its contents are added.
(Files need to be remembered as:
virtual path
real path, possibly in an archive
priority [currently always zero]
priority [currently always zero])
When adding a file, which may have the same virtual path as another file:
If one file has higher priority than the other, use the higher priority one.
If the files are the same (mtime and size), and one is in an archive while the other isn't, use the archived one.
Otherwise, use the most recently added one.
Files are added in breadth-first alphabetic order. When a zip file is encountered in the root directory, all its contents are added.
Otherwise, use the most recently added one. (This should only happen when two archives contain the same file.)
* Clean up the list by removing unnecessary files:
* Clean up the list:
Partly done in the above stage:
Prune CVS and .svn directories
After generating the whole list, tidy up XMBs:
If multiple XMBs exist for an XML file, ignore all but the one which corresponds to the current version of the XML.
* Sort them, using the file-lists that the game generates:
Read all filelist.txt files.
Sort the files based on those lists, using the most recent as the most influential.
* Sort them:
Read all filelist.txt files.
Sort the files based on those lists, using the most recent as the most influential.
* Create the new archive. If it's replacing an older archive built with this tool, delete that older one.
=cut
use File::Find;
use Archive::Zip;
use Class::Struct;
struct( FileData => [
struct( FileData => [
archive => '$', # archive object, or undef if loose
location => '$', # archive member object, or filename if loose
priority => '$' # numeric priority
] );
my $XMB_VERSION = 'A'; # TODO: Find a way of not having to update this manually
generate_archive("../../../binaries", "official");
@ -66,34 +68,37 @@ generate_archive("../../../binaries", "official");
sub generate_archive {
my ($binaries, $modname) = @_;
print "Generating list of files...\n";
my $virtual_files = get_files("$binaries/data/mods/$modname");
# TODO: Tidy up XMBs
print "Tidying file list...\n";
tidy_files($virtual_files);
print "Sorting files...\n";
my $sorted_files = sort_files($virtual_files, "$binaries/logs");
print "Creating archive...\n";
create_archive("$binaries/data/mods/$modname/$modname.zip", $sorted_files);
print "Completed.\n";
}
### File retrieval code ###
# Gets a list of all data files under $root, reading archives and handling priorities
sub get_files {
my ($root) = @_;
my %virtual_files;
# Code that gets called for every file under $root:
my $wanted = sub {
if (/(\.svn|CVS)$/) { # ignore .svn and CVS directories
@ -115,7 +120,7 @@ sub get_files {
# Make sure directories are handled in the correct order
my $preprocess = sub { sort @_ };
find({ wanted => $wanted, preprocess => $preprocess, no_chdir => 1 }, $root);
return \%virtual_files;
@ -133,7 +138,7 @@ sub get_virtual_path {
# Tests whether a file looks like a zip
sub is_zip {
my ($filename) = @_;
# Check for the four-byte header to make sure it's a zip
open my $f, '<', $filename or die "Error opening $filename for input: $!";
binmode $f;
@ -147,7 +152,7 @@ sub add_zip {
my ($filename, $files) = @_;
my $zip = Archive::Zip->new($filename) or die "Error opening zip file $filename";
add_zipped_file($_, $files, $zip) for grep { not $_->isDirectory } $zip->members();
}
@ -155,12 +160,12 @@ sub add_zip {
# Adds a single file from a zip to $files, taking care of varying priorities
sub add_zipped_file {
my ($member, $files, $zip) = @_;
my $virtual_path = $member->fileName;
my $new_file = FileData->new(archive => $zip, location => $member, priority => 0);
my $old_file = $files->{$virtual_path};
if (should_override($new_file, $old_file)) {
$files->{$virtual_path} = $new_file;
}
@ -183,14 +188,14 @@ sub add_normal_file {
# Work out whether $new_file ought to override $old_file, considering priorities and archivedness
sub should_override {
my ($new_file, $old_file) = @_;
# Use the new file if there's no old one
return 1 if not $old_file;
# Higher priority overrides lower priority
return 1 if $new_file->priority > $old_file->priority;
return 0 if $new_file->priority < $old_file->priority;
if (files_equivalent($new_file, $old_file)) {
# Later archives override earlier archives
@ -200,7 +205,7 @@ sub should_override {
return 1 if $new_file->archive;
return 0 if $old_file->archive;
}
# Later files override earlier files
return 1;
}
@ -223,6 +228,50 @@ sub get_size {
### File list tidying code ###
sub tidy_files {
my ($virtual_files) = @_;
# Remove unnecessary XMBs:
for my $filename (keys %$virtual_files) {
# Check if this is an XMB
if ($filename =~ /\.xmb$/i) {
# If so, check for a corresponding XML:
my $xmlfilename = $filename;
if ($xmlfilename =~ s/_([0-9a-f]{16})$XMB_VERSION\.xmb$/.xml/o) {
my $date_size = $1;
if (exists $virtual_files->{$xmlfilename}
and sprintf('%08x%08x', get_mtime($virtual_files->{$xmlfilename}), get_size($virtual_files->{$xmlfilename})) eq $date_size)
{
# It exists - fine
next;
}
}
# No matching XML found, so remove this XMB
delete $virtual_files->{$filename};
}
}
=pod
# Check for XMLs that don't have XMBs:
for my $filename (keys %$virtual_files) {
if ($filename =~ /\.xml$/i) {
my $date_size = sprintf('%08x%08x', get_mtime($virtual_files->{$filename}), get_size($virtual_files->{$filename}));
my $xmbfilename = $filename;
$xmbfilename =~ s/\.xml$/_$date_size$XMB_VERSION.xmb/;
if (not exists $virtual_files->{$xmbfilename}) {
print "No XMB found for $filename\n";
}
}
}
=cut
}
### File list sorting code ###
@ -232,31 +281,31 @@ sub sort_files {
my ($virtual_files, $logsdir) = @_;
my %unhandled_files = %$virtual_files;
my @filelist = read_filelist($logsdir);
my @sorted_files;
# Add files to @sorted_files, in the order they appear in @filelist
for (@filelist) {
if (exists $virtual_files->{$_}) {
push @sorted_files, [ $_, $virtual_files->{$_} ];
delete $virtual_files->{$_};
}
}
}
# Add any remaining files (which weren't referenced by the filelists), just sorted alphabetically
push @sorted_files, map [ $_, $virtual_files->{$_} ], sort keys %$virtual_files;
return \@sorted_files;
}
# Returns a list of files in the order that they're loaded by the engine
sub read_filelist {
my ($logsdir) = @_;
my @files;
# TODO: Multiple file-lists
open my $f, '<', "$logsdir/filelist.txt" or die "Error opening $logsdir/filelist.txt: $!";
while (<$f>) {
@ -275,9 +324,11 @@ sub read_filelist {
# Builds an archive from the list of $files
sub create_archive {
my ($filename, $files) = @_;
# print "$_->[0]\n" for @$files; return;
my $zip = new Archive::Zip;
for my $file (@$files) {
if ($file->[1]->archive) {
$zip->addMember($file->[1]->location) or die "Error adding zipped file $file->[0] to zip";
@ -285,6 +336,6 @@ sub create_archive {
$zip->addFile($file->[1]->location, $file->[0]) or die "Error adding file $file->[0] to zip";
}
}
my $err = $zip->overwriteAs($filename); $err == Archive::Zip::AZ_OK or die "Error saving zip file $filename ($err)";
}