Replace checkrefs.pl by a python script. This makes it easier to run on Windows for non technical persons.
- Add support for tips - Fix other scripts not writing to the correct output (they were writing info messages to stderr) Based on a patch by: @mammadori and @cyrille Differential Revision: https://code.wildfiregames.com/D3213 This was SVN commit r26350.
This commit is contained in:
parent
f17077272f
commit
936fb5a172
@ -1,191 +0,0 @@
|
||||
package Entity;
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use XML::Parser;
|
||||
use Data::Dumper;
|
||||
use File::Find;
|
||||
|
||||
my $vfsroot = '../../../binaries/data/mods';
|
||||
|
||||
sub get_filename
|
||||
{
|
||||
my ($vfspath, $mod) = @_;
|
||||
my $fn = "$vfsroot/$mod/simulation/templates/special/filter/$vfspath.xml";
|
||||
if (not -e $fn) {
|
||||
$fn = "$vfsroot/$mod/simulation/templates/mixins/$vfspath.xml";
|
||||
}
|
||||
if (not -e $fn) {
|
||||
$fn = "$vfsroot/$mod/simulation/templates/$vfspath.xml";
|
||||
}
|
||||
return $fn;
|
||||
}
|
||||
|
||||
sub get_file
|
||||
{
|
||||
my ($vfspath, $mod) = @_;
|
||||
my $fn = get_filename($vfspath, $mod);
|
||||
open my $f, $fn or die "Error loading $fn: $!";
|
||||
local $/;
|
||||
return <$f>;
|
||||
}
|
||||
|
||||
sub trim
|
||||
{
|
||||
my ($t) = @_;
|
||||
return '' if not defined $t;
|
||||
$t =~ /^\s*(.*?)\s*$/s;
|
||||
return $1;
|
||||
}
|
||||
|
||||
sub load_xml
|
||||
{
|
||||
my ($vfspath, $file) = @_;
|
||||
my $root = {};
|
||||
my @stack = ($root);
|
||||
my $p = new XML::Parser(Handlers => {
|
||||
Start => sub {
|
||||
my ($e, $n, %a) = @_;
|
||||
my $t = {};
|
||||
die "Duplicate child node '$n'" if exists $stack[-1]{$n};
|
||||
$stack[-1]{$n} = $t;
|
||||
for (keys %a) {
|
||||
$t->{'@'.$_}{' content'} = trim($a{$_});
|
||||
}
|
||||
push @stack, $t;
|
||||
},
|
||||
End => sub {
|
||||
my ($e, $n) = @_;
|
||||
$stack[-1]{' content'} = trim($stack[-1]{' content'});
|
||||
pop @stack;
|
||||
},
|
||||
Char => sub {
|
||||
my ($e, $str) = @_;
|
||||
$stack[-1]{' content'} .= $str;
|
||||
},
|
||||
});
|
||||
eval {
|
||||
$p->parse($file);
|
||||
};
|
||||
if ($@) {
|
||||
die "Error parsing $vfspath: $@";
|
||||
}
|
||||
return $root;
|
||||
}
|
||||
|
||||
sub apply_layer
|
||||
{
|
||||
my ($base, $new) = @_;
|
||||
if ($new->{'@datatype'} and $new->{'@datatype'}{' content'} eq 'tokens') {
|
||||
my @old = split /\s+/, ($base->{' content'} || '');
|
||||
my @new = split /\s+/, ($new->{' content'} || '');
|
||||
my @t = @old;
|
||||
for my $n (@new) {
|
||||
if ($n =~ /^-(.*)/) {
|
||||
@t = grep $_ ne $1, @t;
|
||||
} else {
|
||||
push @t, $n if not grep $_ eq $n, @t;
|
||||
}
|
||||
}
|
||||
$base->{' content'} = join ' ', @t;
|
||||
} elsif ($new->{'@op'}) {
|
||||
my $op = $new->{'@op'}{' content'};
|
||||
my $op1 = $base->{' content'};
|
||||
my $op2 = $new->{' content'};
|
||||
if ($op eq 'add') {
|
||||
$base->{' content'} = $op1 + $op2;
|
||||
}
|
||||
elsif ($op eq 'mul') {
|
||||
$base->{' content'} = $op1 * $op2;
|
||||
}
|
||||
elsif ($op eq 'mul_round') {
|
||||
# This is incorrect (floors instead of rounding)
|
||||
# but for schema purposes it ought be fine.
|
||||
$base->{' content'} = int($op1 * $op2);
|
||||
}
|
||||
else {
|
||||
die "Invalid operator '$op'";
|
||||
}
|
||||
} else {
|
||||
$base->{' content'} = $new->{' content'};
|
||||
}
|
||||
for my $k (grep $_ ne ' content', keys %$new) {
|
||||
if ($new->{$k}{'@disable'}) {
|
||||
delete $base->{$k};
|
||||
} else {
|
||||
if ($new->{$k}{'@replace'}) {
|
||||
delete $base->{$k};
|
||||
}
|
||||
$base->{$k} ||= {};
|
||||
apply_layer($base->{$k}, $new->{$k});
|
||||
delete $base->{$k}{'@replace'};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub get_main_mod
|
||||
{
|
||||
my ($vfspath, $mods) = @_;
|
||||
my @mods_list = split(/\|/, $mods);
|
||||
my $main_mod = $mods_list[0];
|
||||
my $fn = "$vfsroot/$main_mod/simulation/templates/$vfspath.xml";
|
||||
if (not -e $fn)
|
||||
{
|
||||
for my $dep (@mods_list)
|
||||
{
|
||||
$fn = "$vfsroot/$dep/simulation/templates/$vfspath.xml";
|
||||
if (-e $fn)
|
||||
{
|
||||
$main_mod = $dep;
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
return $main_mod;
|
||||
}
|
||||
|
||||
sub load_inherited
|
||||
{
|
||||
my ($vfspath, $mods, $base) = @_;
|
||||
if ($vfspath =~ /\|/) {
|
||||
my @paths = split(/\|/, $vfspath, 2);
|
||||
$base = load_inherited($paths[1], $mods, $base);
|
||||
$base = load_inherited($paths[0], $mods, $base);
|
||||
return $base
|
||||
}
|
||||
my $main_mod = get_main_mod($vfspath, $mods);
|
||||
my $layer = load_xml($vfspath, get_file($vfspath, $main_mod));
|
||||
|
||||
if ($layer->{Entity}{'@parent'}) {
|
||||
my $parent = load_inherited($layer->{Entity}{'@parent'}{' content'}, $mods, $base);
|
||||
apply_layer($parent->{Entity}, $layer->{Entity});
|
||||
return $parent;
|
||||
} else {
|
||||
if (not $base) {
|
||||
return $layer;
|
||||
}
|
||||
else {
|
||||
apply_layer($base->{Entity}, $layer->{Entity});
|
||||
return $base
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub find_entities
|
||||
{
|
||||
my ($modName) = @_;
|
||||
my @files;
|
||||
my $find_process = sub {
|
||||
return $File::Find::prune = 1 if $_ eq '.svn';
|
||||
my $n = $File::Find::name;
|
||||
return if /~$/;
|
||||
return unless -f $_;
|
||||
$n =~ s~\Q$vfsroot\E/$modName/simulation/templates/~~;
|
||||
$n =~ s/\.xml$//;
|
||||
push @files, $n;
|
||||
};
|
||||
find({ wanted => $find_process }, "$vfsroot/$modName/simulation/templates");
|
||||
|
||||
return @files;
|
||||
}
|
||||
@ -1,715 +0,0 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use Data::Dumper;
|
||||
use File::Find;
|
||||
use XML::Simple;
|
||||
use JSON;
|
||||
use Getopt::Long qw(GetOptions);
|
||||
|
||||
use lib ".";
|
||||
use Entity;
|
||||
|
||||
GetOptions (
|
||||
'--check-unused' => \(my $checkUnused = 0),
|
||||
'--check-map-xml' => \(my $checkMapXml = 0),
|
||||
'--validate-templates' => \(my $validateTemplates = 0),
|
||||
'--mod-to-check=s' => \(my $modToCheck = "public")
|
||||
);
|
||||
|
||||
my @files;
|
||||
my @roots;
|
||||
my @deps;
|
||||
|
||||
# Force and checkMapXml if checkUnused is enabled to avoid false positives.
|
||||
$checkMapXml |= $checkUnused;
|
||||
my $vfsroot = '../../../binaries/data/mods';
|
||||
my $supportedTextureFormats = 'dds|png';
|
||||
my $mods = get_mod_dependencies_string($modToCheck);
|
||||
my $mod_list_string = $modToCheck;
|
||||
if ($mods ne "")
|
||||
{
|
||||
$mod_list_string = $mod_list_string."|$mods";
|
||||
}
|
||||
$mod_list_string = $mod_list_string."|mod";
|
||||
print("Checking $modToCheck\'s integrity. \n");
|
||||
print("The following mod(s) will be loaded: $mod_list_string. \n");
|
||||
my @mods_list = split(/\|/, "$mod_list_string");
|
||||
|
||||
sub get_mod_dependencies
|
||||
{
|
||||
my ($mod) = @_;
|
||||
my $modjson = parse_json_file_full_path("$vfsroot/$mod/mod.json");
|
||||
my $modjsondeps = $modjson->{'dependencies'};
|
||||
for my $dep (@{$modjsondeps})
|
||||
{
|
||||
# 0ad's folder isn't named like the mod.
|
||||
if(index($dep, "0ad") != -1)
|
||||
{
|
||||
$dep = "public";
|
||||
}
|
||||
}
|
||||
|
||||
return $modjsondeps;
|
||||
}
|
||||
|
||||
sub get_mod_dependencies_string
|
||||
{
|
||||
my ($mod) = @_;
|
||||
return join( '|',@{get_mod_dependencies($mod)});
|
||||
}
|
||||
|
||||
sub vfs_to_physical
|
||||
{
|
||||
my ($vfsPath) = @_;
|
||||
my $fn = vfs_to_relative_to_mods($vfsPath);
|
||||
return "$vfsroot/$fn";
|
||||
}
|
||||
|
||||
sub vfs_to_relative_to_mods
|
||||
{
|
||||
my ($vfsPath) = @_;
|
||||
|
||||
for my $dep (@mods_list)
|
||||
{
|
||||
my $fn = "$dep/$vfsPath";
|
||||
|
||||
if (-e "$vfsroot/$fn")
|
||||
{
|
||||
return $fn;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub find_files
|
||||
{
|
||||
my ($vfsPath, $extn) = @_;
|
||||
my @files;
|
||||
my $find_process = sub {
|
||||
return $File::Find::prune = 1 if $_ eq '.svn';
|
||||
my $n = $File::Find::name;
|
||||
return if /~$/;
|
||||
return unless -f $_;
|
||||
return unless /\.($extn)$/;
|
||||
$n =~ s~\Q$vfsroot\E/($mod_list_string)/~~;
|
||||
push @files, $n;
|
||||
};
|
||||
|
||||
for my $dep (@mods_list)
|
||||
{
|
||||
find({ wanted => $find_process },"$vfsroot/$dep/$vfsPath") if -d "$vfsroot/$dep/$vfsPath";
|
||||
}
|
||||
|
||||
return @files;
|
||||
}
|
||||
|
||||
sub parse_json_file_full_path
|
||||
{
|
||||
my ($vfspath) = @_;
|
||||
open my $fh, $vfspath or die "Failed to open '$vfspath': $!";
|
||||
# decode_json expects a UTF-8 string and doesn't handle BOMs, so we strip those
|
||||
# (see http://trac.wildfiregames.com/ticket/1556)
|
||||
return decode_json(do { local $/; my $file = <$fh>; $file =~ s/^\xEF\xBB\xBF//; $file });
|
||||
}
|
||||
|
||||
sub parse_json_file
|
||||
{
|
||||
my ($vfspath) = @_;
|
||||
return parse_json_file_full_path(vfs_to_physical($vfspath))
|
||||
}
|
||||
|
||||
sub add_entities
|
||||
{
|
||||
print "Loading entities...\n";
|
||||
|
||||
my @entfiles = find_files('simulation/templates', 'xml');
|
||||
s~^simulation/templates/(.*)\.xml$~$1~ for @entfiles;
|
||||
|
||||
for my $f (sort @entfiles)
|
||||
{
|
||||
my $path = "simulation/templates/$f.xml";
|
||||
push @files, $path;
|
||||
my $ent = Entity::load_inherited($f, "$mod_list_string");
|
||||
|
||||
if ($ent->{Entity}{'@parent'})
|
||||
{
|
||||
my @parents = split(/\|/, $ent->{Entity}{'@parent'}{' content'});
|
||||
for my $parentPath (@parents)
|
||||
{
|
||||
push @deps, [ $path, "simulation/templates/" . $parentPath . ".xml" ];
|
||||
}
|
||||
}
|
||||
|
||||
if ($f !~ /^template_/)
|
||||
{
|
||||
push @roots, $path;
|
||||
if ($ent->{Entity}{VisualActor} and $ent->{Entity}{VisualActor}{Actor})
|
||||
{
|
||||
my $phenotypes = $ent->{Entity}{Identity}{Phenotype}{' content'} || "default";
|
||||
my @phenotypes = split /\s/,$phenotypes;
|
||||
|
||||
for my $phenotype (@phenotypes)
|
||||
{
|
||||
# See simulation2/components/CCmpVisualActor.cpp and Identity.js for explanation.
|
||||
my $actorPath = $ent->{Entity}{VisualActor}{Actor}{' content'};
|
||||
$actorPath =~ s/{phenotype}/$phenotype/g;
|
||||
push @deps, [ $path, "art/actors/" . $actorPath ];
|
||||
}
|
||||
|
||||
push @deps, [ $path, "art/actors/" . $ent->{Entity}{VisualActor}{FoundationActor}{' content'} ] if $ent->{Entity}{VisualActor}{FoundationActor};
|
||||
}
|
||||
|
||||
if ($ent->{Entity}{Sound})
|
||||
{
|
||||
my $phenotypes = $ent->{Entity}{Identity}{Phenotype}{' content'} || "default";
|
||||
my $lang = $ent->{Entity}{Identity}{Lang}{' content'} || "greek";
|
||||
|
||||
my @phenotypes = split /\s/,$phenotypes;
|
||||
|
||||
for my $phenotype (@phenotypes)
|
||||
{
|
||||
for (grep ref($_), values %{$ent->{Entity}{Sound}{SoundGroups}})
|
||||
{
|
||||
# see simulation/components/Sound.js and Identity.js for explanation
|
||||
my $soundPath = $_->{' content'};
|
||||
$soundPath =~ s/{phenotype}/$phenotype/g;
|
||||
$soundPath =~ s/{lang}/$lang/g;
|
||||
push @deps, [ $path, "audio/" . $soundPath ];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($ent->{Entity}{Identity})
|
||||
{
|
||||
push @deps, [ $path, "art/textures/ui/session/portraits/" . $ent->{Entity}{Identity}{Icon}{' content'} ] if $ent->{Entity}{Identity}{Icon} and $ent->{Entity}{Identity}{Icon}{' content'} ne '';
|
||||
}
|
||||
|
||||
if ($ent->{Entity}{Heal} and $ent->{Entity}{Heal}{RangeOverlay})
|
||||
{
|
||||
push @deps, [ $path, "art/textures/selection/" . $ent->{Entity}{Heal}{RangeOverlay}{LineTexture}{' content'} ] if $ent->{Entity}{Heal}{RangeOverlay}{LineTexture} and $ent->{Entity}{Heal}{RangeOverlay}{LineTexture}{' content'} ne '';
|
||||
push @deps, [ $path, "art/textures/selection/" . $ent->{Entity}{Heal}{RangeOverlay}{LineTextureMask}{' content'} ] if $ent->{Entity}{Heal}{RangeOverlay}{LineTextureMask} and $ent->{Entity}{Heal}{RangeOverlay}{LineTextureMask}{' content'} ne '';
|
||||
}
|
||||
|
||||
if ($ent->{Entity}{Selectable} and $ent->{Entity}{Selectable}{Overlay} and $ent->{Entity}{Selectable}{Overlay}{Texture})
|
||||
{
|
||||
push @deps, [ $path, "art/textures/selection/" . $ent->{Entity}{Selectable}{Overlay}{Texture}{MainTexture}{' content'} ] if $ent->{Entity}{Selectable}{Overlay}{Texture}{MainTexture} and $ent->{Entity}{Selectable}{Overlay}{Texture}{MainTexture}{' content'} ne '';
|
||||
push @deps, [ $path, "art/textures/selection/" . $ent->{Entity}{Selectable}{Overlay}{Texture}{MainTextureMask}{' content'} ] if $ent->{Entity}{Selectable}{Overlay}{Texture}{MainTextureMask} and $ent->{Entity}{Selectable}{Overlay}{Texture}{MainTextureMask}{' content'} ne '';
|
||||
}
|
||||
|
||||
if ($ent->{Entity}{Formation})
|
||||
{
|
||||
push @deps, [ $path, "art/textures/ui/session/icons/" . $ent->{Entity}{Formation}{Icon}{' content'} ] if $ent->{Entity}{Formation}{Icon} and $ent->{Entity}{Formation}{Icon}{' content'} ne '';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub push_variant_dependencies
|
||||
{
|
||||
my ($variant, $f) = @_;
|
||||
push @deps, [ $f, "art/variants/$variant->{file}" ] if $variant->{file};
|
||||
push @deps, [ $f, "art/meshes/$variant->{mesh}" ] if $variant->{mesh};
|
||||
push @deps, [ $f, "art/particles/$variant->{particles}{file}" ] if $variant->{particles}{file};
|
||||
for my $tex (@{$variant->{textures}{texture}})
|
||||
{
|
||||
push @deps, [ $f, "art/textures/skins/$tex->{file}" ] if $tex->{file};
|
||||
}
|
||||
for my $prop (@{$variant->{props}{prop}})
|
||||
{
|
||||
push @deps, [ $f, "art/actors/$prop->{actor}" ] if $prop->{actor};
|
||||
}
|
||||
for my $anim (@{$variant->{animations}{animation}})
|
||||
{
|
||||
push @deps, [ $f, "art/animation/$anim->{file}" ] if $anim->{file};
|
||||
}
|
||||
}
|
||||
|
||||
sub add_actors
|
||||
{
|
||||
print "Loading actors...\n";
|
||||
|
||||
my @actorfiles = find_files('art/actors', 'xml');
|
||||
for my $f (sort @actorfiles)
|
||||
{
|
||||
push @files, $f;
|
||||
push @roots, $f;
|
||||
|
||||
my $actor = XMLin(vfs_to_physical($f), ForceArray => [qw(group variant texture prop animation)], KeyAttr => []) or die "Failed to parse '$f': $!";
|
||||
|
||||
for my $group (@{$actor->{group}})
|
||||
{
|
||||
for my $variant (@{$group->{variant}})
|
||||
{
|
||||
push_variant_dependencies($variant, $f);
|
||||
}
|
||||
}
|
||||
|
||||
push @deps, [ $f, "art/materials/$actor->{material}" ] if $actor->{material};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub add_variants
|
||||
{
|
||||
print "Loading variants...\n";
|
||||
my @variantfiles = find_files('art/variants', 'xml');
|
||||
|
||||
for my $f (sort @variantfiles)
|
||||
{
|
||||
push @files, $f;
|
||||
push @roots, $f;
|
||||
my $variant = XMLin(vfs_to_physical($f), ForceArray => [qw(texture prop animation)], KeyAttr => []) or die "Failed to parse '$f': $!";
|
||||
push_variant_dependencies($variant, $f);
|
||||
}
|
||||
}
|
||||
|
||||
sub add_art
|
||||
{
|
||||
print "Loading art files...\n";
|
||||
push @files, find_files('art/textures/particles', $supportedTextureFormats);
|
||||
push @files, find_files('art/textures/terrain', $supportedTextureFormats);
|
||||
push @files, find_files('art/textures/skins', $supportedTextureFormats);
|
||||
push @files, find_files('art/meshes', 'pmd|dae');
|
||||
push @files, find_files('art/animation', 'psa|dae');
|
||||
}
|
||||
|
||||
sub add_materials
|
||||
{
|
||||
print "Loading materials...\n";
|
||||
my @materialfiles = find_files('art/materials', 'xml');
|
||||
for my $f (sort @materialfiles)
|
||||
{
|
||||
push @files, $f;
|
||||
|
||||
my $material = XMLin(vfs_to_physical($f), ForceArray => [qw(alternative)], KeyAttr => []);
|
||||
for my $alternative (@{$material->{alternative}})
|
||||
{
|
||||
push @deps, [ $f, "art/materials/$alternative->{material}" ] if $alternative->{material};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub add_particles
|
||||
{
|
||||
print "Loading particles...\n";
|
||||
my @particlefiles = find_files('art/particles', 'xml');
|
||||
for my $f (sort @particlefiles)
|
||||
{
|
||||
push @files, $f;
|
||||
|
||||
my $particle = XMLin(vfs_to_physical($f));
|
||||
push @deps, [ $f, "$particle->{texture}" ] if $particle->{texture};
|
||||
}
|
||||
}
|
||||
|
||||
sub add_maps_xml
|
||||
{
|
||||
print "Loading maps XML...\n";
|
||||
my @mapfiles = find_files('maps/scenarios', 'xml');
|
||||
push @mapfiles, find_files('maps/skirmishes', 'xml');
|
||||
push @mapfiles, find_files('maps/tutorials', 'xml');
|
||||
for my $f (sort @mapfiles)
|
||||
{
|
||||
push @files, $f;
|
||||
push @roots, $f;
|
||||
|
||||
my $map = XMLin(vfs_to_physical($f), ForceArray => [qw(Entity)], KeyAttr => []) or die "Failed to parse '$f': $!";
|
||||
|
||||
my %used;
|
||||
for my $entity (@{$map->{Entities}{Entity}})
|
||||
{
|
||||
$used{$entity->{Template}} = 1;
|
||||
}
|
||||
|
||||
for my $template (keys %used)
|
||||
{
|
||||
if ($template =~ /^actor\|(.*)$/)
|
||||
{
|
||||
# Handle special 'actor|' case
|
||||
push @deps, [ $f, "art/actors/$1" ];
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($template =~ /^resource\|(.*)$/)
|
||||
{
|
||||
# Handle special 'resource|' case
|
||||
$template = $1;
|
||||
}
|
||||
push @deps, [ $f, "simulation/templates/$template.xml" ];
|
||||
}
|
||||
}
|
||||
|
||||
# Map previews
|
||||
my $settings = decode_json($map->{ScriptSettings});
|
||||
push @deps, [ $f, "art/textures/ui/session/icons/mappreview/" . $settings->{Preview} ] if $settings->{Preview};
|
||||
}
|
||||
}
|
||||
|
||||
sub add_maps_pmp
|
||||
{
|
||||
print "Loading maps PMP...\n";
|
||||
|
||||
# Need to generate terrain texture filename=>path lookup first
|
||||
my %terrains;
|
||||
for my $f (find_files('art/terrains', 'xml'))
|
||||
{
|
||||
$f =~ /([^\/]+)\.xml/ or die;
|
||||
|
||||
# ignore terrains.xml
|
||||
if ($f !~ /terrains.xml$/)
|
||||
{
|
||||
warn "Duplicate terrain name '$1' (from '$terrains{$1}' and '$f')\n" if $terrains{$1};
|
||||
$terrains{$1} = $f;
|
||||
}
|
||||
}
|
||||
|
||||
my @mapfiles = find_files('maps/scenarios', 'pmp');
|
||||
push @mapfiles, find_files('maps/skirmishes', 'pmp');
|
||||
for my $f (sort @mapfiles)
|
||||
{
|
||||
push @files, $f;
|
||||
|
||||
push @roots, $f;
|
||||
|
||||
open my $fh, vfs_to_physical($f) or die "Failed to open '$f': $!";
|
||||
binmode $fh;
|
||||
|
||||
my $buf;
|
||||
|
||||
read $fh, $buf, 4;
|
||||
die "Invalid PMP header ($buf) in '$f'" unless $buf eq "PSMP";
|
||||
|
||||
read $fh, $buf, 4;
|
||||
my $version = unpack 'V', $buf;
|
||||
die "Invalid PMP version ($version) in '$f'" unless $version == 7;
|
||||
|
||||
read $fh, $buf, 4;
|
||||
my $datasize = unpack 'V', $buf;
|
||||
|
||||
read $fh, $buf, 4;
|
||||
my $mapsize = unpack 'V', $buf;
|
||||
|
||||
seek $fh, 2 * ($mapsize*16+1)*($mapsize*16+1), 1; # heightmap
|
||||
|
||||
read $fh, $buf, 4;
|
||||
my $numtexs = unpack 'V', $buf;
|
||||
|
||||
for (0..$numtexs-1)
|
||||
{
|
||||
read $fh, $buf, 4;
|
||||
my $len = unpack 'V', $buf;
|
||||
my $str;
|
||||
read $fh, $str, $len;
|
||||
|
||||
push @deps, [ $f, $terrains{$str} || "art/terrains/(unknown)/$str" ];
|
||||
}
|
||||
|
||||
# ignore patches data
|
||||
}
|
||||
}
|
||||
|
||||
sub add_soundgroups
|
||||
{
|
||||
print "Loading sound groups...\n";
|
||||
my @soundfiles = find_files('audio', 'xml');
|
||||
for my $f (sort @soundfiles)
|
||||
{
|
||||
push @files, $f;
|
||||
push @roots, $f;
|
||||
|
||||
my $sound = XMLin(vfs_to_physical($f), ForceArray => [qw(Sound)], KeyAttr => []) or die "Failed to parse '$f': $!";
|
||||
|
||||
my $path = $sound->{Path};
|
||||
$path =~ s/\/$//; # strip optional trailing slash
|
||||
|
||||
for (@{$sound->{Sound}})
|
||||
{
|
||||
push @deps, [$f, "$path/$_" ];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub add_audio
|
||||
{
|
||||
print "Loading audio files...\n";
|
||||
push @files, find_files('audio', 'ogg');
|
||||
}
|
||||
|
||||
sub add_gui_xml
|
||||
{
|
||||
print "Loading GUI XML...\n";
|
||||
my @guifiles = find_files('gui', 'xml');
|
||||
for my $f (sort @guifiles)
|
||||
{
|
||||
push @files, $f;
|
||||
|
||||
# GUI page definitions are assumed to be named page[_something].xml and alone in that.
|
||||
if ($f =~ /\/page(_[^.\/]+)?\.xml$/)
|
||||
{
|
||||
push @roots, $f;
|
||||
my $xml = XMLin(vfs_to_physical($f), ForceArray => [qw(include)], KeyAttr => []) or die "Failed to parse '$f': $!";
|
||||
|
||||
for my $include (@{$xml->{include}})
|
||||
{
|
||||
# If including an entire directory, find all the *.xml files
|
||||
if ($include =~ /\/$/)
|
||||
{
|
||||
push @deps, [ $f, $_ ] for find_files("gui/$include", 'xml');
|
||||
}
|
||||
else
|
||||
{
|
||||
push @deps, [ $f, "gui/$include" ];
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
my $xml = XMLin(vfs_to_physical($f), ForceArray => [qw(object script action sprite image)], KeyAttr => [], KeepRoot => 1) or die "Failed to parse '$f': $!";
|
||||
my $name = (keys %$xml)[0];
|
||||
if ($name eq 'objects' or $name eq 'object')
|
||||
{
|
||||
for (grep ref $_ , @{$xml->{objects}{script}})
|
||||
{
|
||||
push @deps, [ $f, $_->{file} ] if $_->{file};
|
||||
if ($_->{directory})
|
||||
{
|
||||
# If including an entire directory, find all the *.js files
|
||||
push @deps, [ $f, $_ ] for find_files($_->{directory}, 'js')
|
||||
}
|
||||
}
|
||||
my $add_objects;
|
||||
$add_objects = sub
|
||||
{
|
||||
my ($parent) = @_;
|
||||
for my $obj (@{$parent->{object}})
|
||||
{
|
||||
# TODO: look at sprites, styles, etc
|
||||
$add_objects->($obj);
|
||||
}
|
||||
};
|
||||
$add_objects->($xml->{objects});
|
||||
}
|
||||
elsif ($name eq 'setup')
|
||||
{
|
||||
# TODO: look at sprites, styles, etc
|
||||
}
|
||||
elsif ($name eq 'styles')
|
||||
{
|
||||
# TODO: look at sprites, styles, etc
|
||||
}
|
||||
elsif ($name eq 'sprites')
|
||||
{
|
||||
for my $sprite (@{$xml->{sprites}{sprite}})
|
||||
{
|
||||
for my $image (@{$sprite->{image}})
|
||||
{
|
||||
push @deps, [ $f, "art/textures/ui/$image->{texture}" ] if $image->{texture};
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
print "Unexpected GUI XML root element '$name':\n" . Dumper $xml;
|
||||
exit;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub add_gui_data
|
||||
{
|
||||
print "Loading GUI data...\n";
|
||||
push @files, find_files('gui', 'js');
|
||||
push @files, find_files('art/textures/ui', $supportedTextureFormats);
|
||||
push @files, find_files('art/textures/selection', $supportedTextureFormats);
|
||||
}
|
||||
|
||||
sub add_civs
|
||||
{
|
||||
print "Loading civs...\n";
|
||||
|
||||
my @civfiles = find_files('simulation/data/civs', 'json');
|
||||
for my $f (sort @civfiles)
|
||||
{
|
||||
push @files, $f;
|
||||
|
||||
push @roots, $f;
|
||||
|
||||
my $civ = parse_json_file($f);
|
||||
|
||||
push @deps, [ $f, "audio/music/" . $_->{File} ] for @{$civ->{Music}};
|
||||
}
|
||||
}
|
||||
|
||||
sub add_rms
|
||||
{
|
||||
print "Loading random maps...\n";
|
||||
|
||||
push @files, find_files('maps/random', 'js');
|
||||
my @rmsdefs = find_files('maps/random', 'json');
|
||||
|
||||
for my $f (sort @rmsdefs)
|
||||
{
|
||||
next if $f =~ /^maps\/random\/rmbiome/;
|
||||
|
||||
push @files, $f;
|
||||
|
||||
push @roots, $f;
|
||||
|
||||
my $rms = parse_json_file($f);
|
||||
push @deps, [ $f, "maps/random/" . $rms->{settings}{Script} ] if $rms->{settings}{Script};
|
||||
|
||||
# Map previews
|
||||
push @deps, [ $f, "art/textures/ui/session/icons/mappreview/" . $rms->{settings}{Preview} ] if $rms->{settings}{Preview};
|
||||
}
|
||||
}
|
||||
|
||||
sub add_techs
|
||||
{
|
||||
print "Loading techs...\n";
|
||||
|
||||
my @techfiles = find_files('simulation/data/technologies', 'json');
|
||||
for my $f (sort @techfiles)
|
||||
{
|
||||
push @files, $f;
|
||||
push @roots, $f;
|
||||
|
||||
my $tech = parse_json_file($f);
|
||||
|
||||
push @deps, [ $f, "art/textures/ui/session/portraits/technologies/" . $tech->{icon} ] if $tech->{icon};
|
||||
push @deps, [ $f, "simulation/data/technologies/" . $tech->{supersedes} . ".json" ] if $tech->{supersedes};
|
||||
}
|
||||
}
|
||||
|
||||
sub add_auras
|
||||
{
|
||||
print "Loading auras...\n";
|
||||
|
||||
my @aurafiles = find_files('simulation/data/auras', 'json');
|
||||
for my $f (sort @aurafiles)
|
||||
{
|
||||
push @files, $f;
|
||||
push @roots, $f;
|
||||
|
||||
my $aura = parse_json_file($f);
|
||||
|
||||
push @deps, [ $f, $aura->{overlayIcon} ] if $aura->{overlayIcon};
|
||||
|
||||
if($aura->{rangeOverlay})
|
||||
{
|
||||
push @deps, [ $f, "art/textures/selection/" . $aura->{rangeOverlay}{lineTexture} ] if $aura->{rangeOverlay}{lineTexture};
|
||||
push @deps, [ $f, "art/textures/selection/" . $aura->{rangeOverlay}{lineTextureMask} ] if $aura->{rangeOverlay}{lineTextureMask};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub add_terrains
|
||||
{
|
||||
print "Loading terrains...\n";
|
||||
|
||||
my @terrains = find_files('art/terrains', 'xml');
|
||||
for my $f (sort @terrains)
|
||||
{
|
||||
# ignore terrains.xml
|
||||
if ($f !~ /terrains.xml$/)
|
||||
{
|
||||
push @files, $f;
|
||||
push @roots, $f;
|
||||
|
||||
my $terrain = XMLin(vfs_to_physical($f), ForceArray => [qw(texture)], KeyAttr => []) or die "Failed to parse '$f': $!";
|
||||
|
||||
for my $texture (@{$terrain->{textures}{texture}})
|
||||
{
|
||||
push @deps, [ $f, "art/textures/terrain/$texture->{file}" ] if $texture->{file};
|
||||
}
|
||||
push @deps, [ $f, "art/materials/$terrain->{material}" ] if $terrain->{material};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub check_deps
|
||||
{
|
||||
my %files;
|
||||
@files{@files} = ();
|
||||
|
||||
my %lcfiles;
|
||||
@lcfiles{map lc($_), @files} = @files;
|
||||
|
||||
my %revdeps;
|
||||
for my $d (@deps)
|
||||
{
|
||||
push @{$revdeps{$d->[1]}}, $d->[0];
|
||||
}
|
||||
|
||||
for my $f (sort keys %revdeps)
|
||||
{
|
||||
if ($f =~ /simulation\/templates\//)
|
||||
{
|
||||
next if exists $files{$f =~ s/templates\//templates\/special\/filter\//r};
|
||||
next if exists $files{$f =~ s/templates\//templates\/mixins\//r};
|
||||
}
|
||||
next if exists $files{$f};
|
||||
warn "Missing file '$f' referenced by: " . (join ', ', map "'$_'", map vfs_to_relative_to_mods($_), sort @{$revdeps{$f}}) . "\n";
|
||||
|
||||
if (exists $lcfiles{lc $f})
|
||||
{
|
||||
warn "### Case-insensitive match (found '$lcfiles{lc $f}')\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub check_unused
|
||||
{
|
||||
my %reachable;
|
||||
@reachable{@roots} = ();
|
||||
|
||||
my %deps;
|
||||
for my $d (@deps)
|
||||
{
|
||||
push @{$deps{$d->[0]}}, $d->[1];
|
||||
}
|
||||
|
||||
while (1)
|
||||
{
|
||||
my @newreachable;
|
||||
for my $r (keys %reachable)
|
||||
{
|
||||
push @newreachable, grep { not exists $reachable{$_} } @{$deps{$r}};
|
||||
}
|
||||
last if @newreachable == 0;
|
||||
@reachable{@newreachable} = ();
|
||||
}
|
||||
|
||||
for my $f (sort @files)
|
||||
{
|
||||
next if exists $reachable{$f}
|
||||
|| index($f, "art/terrains/") != -1
|
||||
|| index($f, "maps/random/") != -1
|
||||
|| index($f, "art/materials/") != -1;
|
||||
warn "Unused file '" . vfs_to_relative_to_mods($f) . "'\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
add_maps_xml() if $checkMapXml;
|
||||
add_maps_pmp();
|
||||
add_entities();
|
||||
add_actors();
|
||||
add_variants();
|
||||
add_art();
|
||||
add_materials();
|
||||
add_particles();
|
||||
add_soundgroups();
|
||||
add_audio();
|
||||
add_gui_xml();
|
||||
add_gui_data();
|
||||
add_civs();
|
||||
add_rms();
|
||||
add_techs();
|
||||
add_terrains();
|
||||
add_auras();
|
||||
|
||||
check_deps();
|
||||
check_unused() if $checkUnused;
|
||||
print "\n" if $checkUnused;
|
||||
system("perl ../xmlvalidator/validate.pl") if $validateTemplates;
|
||||
605
source/tools/entity/checkrefs.py
Normal file
605
source/tools/entity/checkrefs.py
Normal file
@ -0,0 +1,605 @@
|
||||
#!/usr/bin/env python3
|
||||
from argparse import ArgumentParser
|
||||
from io import BytesIO
|
||||
from json import load, loads
|
||||
from pathlib import Path
|
||||
from re import split, match
|
||||
from struct import unpack, calcsize
|
||||
from os.path import sep, exists, basename
|
||||
from xml.etree import ElementTree
|
||||
import sys
|
||||
from scriptlib import SimulTemplateEntity, find_files
|
||||
from logging import WARNING, getLogger, StreamHandler, INFO, Formatter, Filter
|
||||
|
||||
class SingleLevelFilter(Filter):
|
||||
def __init__(self, passlevel, reject):
|
||||
self.passlevel = passlevel
|
||||
self.reject = reject
|
||||
|
||||
def filter(self, record):
|
||||
if self.reject:
|
||||
return (record.levelno != self.passlevel)
|
||||
else:
|
||||
return (record.levelno == self.passlevel)
|
||||
|
||||
class CheckRefs:
|
||||
def __init__(self):
|
||||
# list of relative root file:str
|
||||
self.files = []
|
||||
# list of relative file:str
|
||||
self.roots = []
|
||||
# list of tuple (parent_file:str, dep_file:str)
|
||||
self.deps = []
|
||||
self.vfs_root = Path(__file__).resolve().parents[3] / 'binaries' / 'data' / 'mods'
|
||||
self.supportedTextureFormats = ('dds', 'png')
|
||||
self.supportedMeshesFormats = ('pmd', 'dae')
|
||||
self.supportedAnimationFormats = ('psa', 'dae')
|
||||
self.supportedAudioFormats = ('ogg')
|
||||
self.mods = []
|
||||
self.__init_logger
|
||||
|
||||
@property
|
||||
def __init_logger(self):
|
||||
logger = getLogger(__name__)
|
||||
logger.setLevel(INFO)
|
||||
# create a console handler, seems nicer to Windows and for future uses
|
||||
ch = StreamHandler(sys.stdout)
|
||||
ch.setLevel(INFO)
|
||||
ch.setFormatter(Formatter('%(levelname)s - %(message)s'))
|
||||
f1 = SingleLevelFilter(INFO, False)
|
||||
ch.addFilter(f1)
|
||||
logger.addHandler(ch)
|
||||
errorch = StreamHandler(sys.stderr)
|
||||
errorch.setLevel(WARNING)
|
||||
errorch.setFormatter(Formatter('%(levelname)s - %(message)s'))
|
||||
logger.addHandler(errorch)
|
||||
self.logger = logger
|
||||
|
||||
def main(self):
|
||||
ap = ArgumentParser(description="Checks the game files for missing dependencies, unused files,"
|
||||
" and for file integrity.")
|
||||
ap.add_argument('-u', '--check-unused', action='store_true',
|
||||
help="check for all the unused files in the given mods and their dependencies."
|
||||
" Implies --check-map-xml. Currently yields a lot of false positives.")
|
||||
ap.add_argument('-x', '--check-map-xml', action='store_true',
|
||||
help="check maps for missing actor and templates.")
|
||||
ap.add_argument('-a', '--validate-actors', action='store_true',
|
||||
help="run the validator.py script to check if the actors files have extra or missing textures."
|
||||
" This currently only works for the public mod.")
|
||||
ap.add_argument('-t', '--validate-templates', action='store_true',
|
||||
help="run the validator.py script to check if the xml files match their (.rng) grammar file.")
|
||||
ap.add_argument('-m', '--mods', metavar="MOD", dest='mods', nargs='+', default=['public'],
|
||||
help="specify which mods to check. Default to public.")
|
||||
args = ap.parse_args()
|
||||
# force check_map_xml if check_unused is used to avoid false positives.
|
||||
args.check_map_xml |= args.check_unused
|
||||
# ordered uniq mods (dict maintains ordered keys from python 3.6)
|
||||
self.mods = list(dict.fromkeys([*args.mods, *self.get_mod_dependencies(*args.mods), 'mod']).keys())
|
||||
self.logger.info(f"Checking {'|'.join(args.mods)}'s integrity.")
|
||||
self.logger.info(f"The following mods will be loaded: {'|'.join(self.mods)}.")
|
||||
if args.check_map_xml:
|
||||
self.add_maps_xml()
|
||||
self.add_maps_pmp()
|
||||
self.add_entities()
|
||||
self.add_actors()
|
||||
self.add_variants()
|
||||
self.add_art()
|
||||
self.add_materials()
|
||||
self.add_particles()
|
||||
self.add_soundgroups()
|
||||
self.add_audio()
|
||||
self.add_gui_xml()
|
||||
self.add_gui_data()
|
||||
self.add_civs()
|
||||
self.add_rms()
|
||||
self.add_techs()
|
||||
self.add_terrains()
|
||||
self.add_auras()
|
||||
self.add_tips()
|
||||
self.check_deps()
|
||||
if args.check_unused:
|
||||
self.check_unused()
|
||||
if args.validate_templates:
|
||||
sys.path.append("../xmlvalidator/")
|
||||
from validate_grammar import RelaxNGValidator
|
||||
validate = RelaxNGValidator(self.vfs_root, self.mods)
|
||||
validate.run()
|
||||
if args.validate_actors:
|
||||
sys.path.append("../xmlvalidator/")
|
||||
from validator import Validator
|
||||
validator = Validator(self.vfs_root, self.mods)
|
||||
validator.run()
|
||||
|
||||
def get_mod_dependencies(self, *mods):
|
||||
modjsondeps = []
|
||||
for mod in mods:
|
||||
mod_json_path = self.vfs_root / mod / 'mod.json'
|
||||
if not exists(mod_json_path):
|
||||
continue
|
||||
|
||||
with open(mod_json_path, encoding='utf-8') as f:
|
||||
modjson = load(f)
|
||||
# 0ad's folder isn't named like the mod.
|
||||
modjsondeps.extend(['public' if '0ad' in dep else dep for dep in modjson.get('dependencies', [])])
|
||||
return modjsondeps
|
||||
|
||||
def vfs_to_relative_to_mods(self, vfs_path):
|
||||
for dep in self.mods:
|
||||
fn = Path(dep) / vfs_path
|
||||
if (self.vfs_root / fn).exists():
|
||||
return fn
|
||||
return None
|
||||
|
||||
def vfs_to_physical(self, vfs_path):
|
||||
fn = self.vfs_to_relative_to_mods(vfs_path)
|
||||
return self.vfs_root / fn
|
||||
|
||||
def find_files(self, vfs_path, *ext_list):
|
||||
return find_files(self.vfs_root, self.mods, vfs_path, *ext_list)
|
||||
|
||||
def add_maps_xml(self):
|
||||
self.logger.info("Loading maps XML...")
|
||||
mapfiles = self.find_files('maps/scenarios', 'xml')
|
||||
mapfiles.extend(self.find_files('maps/skirmishes', 'xml'))
|
||||
mapfiles.extend(self.find_files('maps/tutorials', 'xml'))
|
||||
actor_prefix = 'actor|'
|
||||
resource_prefix = 'resource|'
|
||||
for (fp, ffp) in sorted(mapfiles):
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
et_map = ElementTree.parse(ffp).getroot()
|
||||
entities = et_map.find('Entities')
|
||||
used = {entity.find('Template').text.strip() for entity in entities.findall('Entity')} if entities is not None else {}
|
||||
for template in used:
|
||||
if template.startswith(actor_prefix):
|
||||
self.deps.append((str(fp), f'art/actors/{template[len(actor_prefix):]}'))
|
||||
elif template.startswith(resource_prefix):
|
||||
self.deps.append((str(fp), f'simulation/templates/{template[len(resource_prefix):]}.xml'))
|
||||
else:
|
||||
self.deps.append((str(fp), f'simulation/templates/{template}.xml'))
|
||||
# Map previews
|
||||
settings = loads(et_map.find('ScriptSettings').text)
|
||||
if settings.get('Preview', None):
|
||||
self.deps.append((str(fp), f'art/textures/ui/session/icons/mappreview/{settings["Preview"]}'))
|
||||
|
||||
def add_maps_pmp(self):
|
||||
self.logger.info("Loading maps PMP...")
|
||||
# Need to generate terrain texture filename=>relative path lookup first
|
||||
terrains = dict()
|
||||
for (fp, ffp) in self.find_files('art/terrains', 'xml'):
|
||||
name = fp.stem
|
||||
# ignore terrains.xml
|
||||
if name != 'terrains':
|
||||
if name in terrains:
|
||||
self.logger.warning(f"Duplicate terrain name '{name}' (from '{terrains[name]}' and '{ffp}')")
|
||||
terrains[name] = str(fp)
|
||||
mapfiles = self.find_files('maps/scenarios', 'pmp')
|
||||
mapfiles.extend(self.find_files('maps/skirmishes', 'pmp'))
|
||||
for (fp, ffp) in sorted(mapfiles):
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
with open(ffp, 'rb') as f:
|
||||
expected_header = b'PSMP'
|
||||
header = f.read(len(expected_header))
|
||||
if header != expected_header:
|
||||
raise ValueError(f"Invalid PMP header {header} in '{ffp}'")
|
||||
int_fmt = '<L' # little endian long int
|
||||
int_len = calcsize(int_fmt)
|
||||
version, = unpack(int_fmt, f.read(int_len))
|
||||
if version != 7:
|
||||
raise ValueError(f"Invalid PMP version ({version}) in '{ffp}'")
|
||||
datasize, = unpack(int_fmt, f.read(int_len))
|
||||
mapsize, = unpack(int_fmt, f.read(int_len))
|
||||
f.seek(2 * (mapsize * 16 + 1) * (mapsize * 16 + 1), 1) # skip heightmap
|
||||
numtexs, = unpack(int_fmt, f.read(int_len))
|
||||
for i in range(numtexs):
|
||||
length, = unpack(int_fmt, f.read(int_len))
|
||||
terrain_name = f.read(length).decode('ascii') # suppose ascii encoding
|
||||
self.deps.append((str(fp), terrains.get(terrain_name, f'art/terrains/(unknown)/{terrain_name}')))
|
||||
|
||||
def add_entities(self):
|
||||
self.logger.info("Loading entities...")
|
||||
simul_templates_path = Path('simulation/templates')
|
||||
# TODO: We might want to get computed templates through the RL interface instead of computing the values ourselves.
|
||||
simul_template_entity = SimulTemplateEntity(self.vfs_root, self.logger)
|
||||
for (fp, _) in sorted(self.find_files(simul_templates_path, 'xml')):
|
||||
self.files.append(str(fp))
|
||||
entity = simul_template_entity.load_inherited(simul_templates_path, str(fp.relative_to(simul_templates_path)), self.mods)
|
||||
if entity.get('parent'):
|
||||
for parent in entity.get('parent').split('|'):
|
||||
self.deps.append((str(fp), str(simul_templates_path / (parent + '.xml'))))
|
||||
if not str(fp).startswith('template_'):
|
||||
self.roots.append(str(fp))
|
||||
if entity and entity.find('VisualActor') is not None and entity.find('VisualActor').find('Actor') is not None:
|
||||
if entity.find('Identity'):
|
||||
phenotype_tag = entity.find('Identity').find('Phenotype')
|
||||
phenotypes = split(r'\s', phenotype_tag.text if phenotype_tag is not None and phenotype_tag.text else 'default')
|
||||
actor = entity.find('VisualActor').find('Actor')
|
||||
if '{phenotype}' in actor.text:
|
||||
for phenotype in phenotypes:
|
||||
# See simulation2/components/CCmpVisualActor.cpp and Identity.js for explanation.
|
||||
actor_path = actor.text.replace('{phenotype}', phenotype)
|
||||
self.deps.append((str(fp), f'art/actors/{actor_path}'))
|
||||
else:
|
||||
actor_path = actor.text
|
||||
self.deps.append((str(fp), f'art/actors/{actor_path}'))
|
||||
foundation_actor = entity.find('VisualActor').find('FoundationActor')
|
||||
if foundation_actor:
|
||||
self.deps.append((str(fp), f'art/actors/{foundation_actor.text}'))
|
||||
if entity.find('Sound'):
|
||||
phenotype_tag = entity.find('Identity').find('Phenotype')
|
||||
phenotypes = split(r'\s', phenotype_tag.text if phenotype_tag is not None and phenotype_tag.text else 'default')
|
||||
lang_tag = entity.find('Identity').find('Lang')
|
||||
lang = lang_tag.text if lang_tag is not None and lang_tag.text else 'greek'
|
||||
sound_groups = entity.find('Sound').find('SoundGroups')
|
||||
for sound_group in sound_groups:
|
||||
if sound_group.text and sound_group.text.strip():
|
||||
if '{phenotype}' in sound_group.text:
|
||||
for phenotype in phenotypes:
|
||||
# see simulation/components/Sound.js and Identity.js for explanation
|
||||
sound_path = sound_group.text.replace('{phenotype}', phenotype).replace('{lang}', lang)
|
||||
self.deps.append((str(fp), f'audio/{sound_path}'))
|
||||
else:
|
||||
sound_path = sound_group.text.replace('{lang}', lang)
|
||||
self.deps.append((str(fp), f'audio/{sound_path}'))
|
||||
if entity.find('Identity') is not None:
|
||||
icon = entity.find('Identity').find('Icon')
|
||||
if icon is not None and icon.text:
|
||||
self.deps.append((str(fp), f'art/textures/ui/session/portraits/{icon.text}'))
|
||||
if entity.find('Heal') is not None and entity.find('Heal').find('RangeOverlay') is not None:
|
||||
range_overlay = entity.find('Heal').find('RangeOverlay')
|
||||
for tag in ('LineTexture', 'LineTextureMask'):
|
||||
elem = range_overlay.find(tag)
|
||||
if elem is not None and elem.text:
|
||||
self.deps.append((str(fp), f'art/textures/selection/{elem.text}'))
|
||||
if entity.find('Selectable') is not None and entity.find('Selectable').find('Overlay') is not None \
|
||||
and entity.find('Selectable').find('Overlay').find('Texture') is not None:
|
||||
texture = entity.find('Selectable').find('Overlay').find('Texture')
|
||||
for tag in ('MainTexture', 'MainTextureMask'):
|
||||
elem = texture.find(tag)
|
||||
if elem is not None and elem.text:
|
||||
self.deps.append((str(fp), f'art/textures/selection/{elem.text}'))
|
||||
if entity.find('Formation') is not None:
|
||||
icon = entity.find('Formation').find('Icon')
|
||||
if icon is not None and icon.text:
|
||||
self.deps.append((str(fp), f'art/textures/ui/session/icons/{icon.text}'))
|
||||
|
||||
def append_variant_dependencies(self, variant, fp):
|
||||
variant_file = variant.get('file')
|
||||
mesh = variant.find('mesh')
|
||||
particles = variant.find('particles')
|
||||
texture_files = [tex.get('file') for tex in variant.find('textures').findall('texture')] \
|
||||
if variant.find('textures') is not None else []
|
||||
prop_actors = [prop.get('actor') for prop in variant.find('props').findall('prop')] \
|
||||
if variant.find('props') is not None else []
|
||||
animation_files = [anim.get('file') for anim in variant.find('animations').findall('animation')] \
|
||||
if variant.find('animations') is not None else []
|
||||
if variant_file:
|
||||
self.deps.append((str(fp), f'art/variants/{variant_file}'))
|
||||
if mesh is not None and mesh.text:
|
||||
self.deps.append((str(fp), f'art/meshes/{mesh.text}'))
|
||||
if particles is not None and particles.get('file'):
|
||||
self.deps.append((str(fp), f'art/particles/{particles.get("file")}'))
|
||||
for texture_file in [x for x in texture_files if x]:
|
||||
self.deps.append((str(fp), f'art/textures/skins/{texture_file}'))
|
||||
for prop_actor in [x for x in prop_actors if x]:
|
||||
self.deps.append((str(fp), f'art/actors/{prop_actor}'))
|
||||
for animation_file in [x for x in animation_files if x]:
|
||||
self.deps.append((str(fp), f'art/animation/{animation_file}'))
|
||||
|
||||
def append_actor_dependencies(self, actor, fp):
|
||||
for group in actor.findall('group'):
|
||||
for variant in group.findall('variant'):
|
||||
self.append_variant_dependencies(variant, fp)
|
||||
material = actor.find('material')
|
||||
if material is not None and material.text:
|
||||
self.deps.append((str(fp), f'art/materials/{material.text}'))
|
||||
|
||||
def add_actors(self):
|
||||
self.logger.info("Loading actors...")
|
||||
for (fp, ffp) in sorted(self.find_files('art/actors', 'xml')):
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
root = ElementTree.parse(ffp).getroot()
|
||||
if root.tag == 'actor':
|
||||
self.append_actor_dependencies(root, fp)
|
||||
|
||||
# model has lods
|
||||
elif root.tag == 'qualitylevels':
|
||||
qualitylevels = root
|
||||
for actor in qualitylevels.findall('actor'):
|
||||
self.append_actor_dependencies(actor, fp)
|
||||
for actor in qualitylevels.findall('inline'):
|
||||
self.append_actor_dependencies(actor, fp)
|
||||
|
||||
def add_variants(self):
|
||||
self.logger.info("Loading variants...")
|
||||
for (fp, ffp) in sorted(self.find_files('art/variants', 'xml')):
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
variant = ElementTree.parse(ffp).getroot()
|
||||
self.append_variant_dependencies(variant, fp)
|
||||
|
||||
def add_art(self):
|
||||
self.logger.info("Loading art files...")
|
||||
self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/particles', *self.supportedTextureFormats)])
|
||||
self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/terrain', *self.supportedTextureFormats)])
|
||||
self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/skins', *self.supportedTextureFormats)])
|
||||
self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/meshes', *self.supportedMeshesFormats)])
|
||||
self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/animation', *self.supportedAnimationFormats)])
|
||||
|
||||
|
||||
def add_materials(self):
|
||||
self.logger.info("Loading materials...")
|
||||
for (fp, ffp) in sorted(self.find_files('art/materials', 'xml')):
|
||||
self.files.append(str(fp))
|
||||
material_elem = ElementTree.parse(ffp).getroot()
|
||||
for alternative in material_elem.findall('alternative'):
|
||||
material = alternative.get('material')
|
||||
if material:
|
||||
self.deps.append((str(fp), f'art/materials/{material}'))
|
||||
|
||||
def add_particles(self):
|
||||
self.logger.info("Loading particles...")
|
||||
for (fp, ffp) in sorted(self.find_files('art/particles', 'xml')):
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
particle = ElementTree.parse(ffp).getroot()
|
||||
texture = particle.find('texture')
|
||||
if texture:
|
||||
self.deps.append((str(fp), texture.text))
|
||||
|
||||
def add_soundgroups(self):
|
||||
self.logger.info("Loading sound groups...")
|
||||
for (fp, ffp) in sorted(self.find_files('audio', 'xml')):
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
sound_group = ElementTree.parse(ffp).getroot()
|
||||
path = sound_group.find('Path').text.rstrip('/')
|
||||
for sound in sound_group.findall('Sound'):
|
||||
self.deps.append((str(fp), f'{path}/{sound.text}'))
|
||||
|
||||
def add_audio(self):
|
||||
self.logger.info("Loading audio files...")
|
||||
self.files.extend([str(fp) for (fp, ffp) in self.find_files('audio/', self.supportedAudioFormats)])
|
||||
|
||||
|
||||
def add_gui_object_repeat(self, obj, fp):
|
||||
for repeat in obj.findall('repeat'):
|
||||
for sub_obj in repeat.findall('object'):
|
||||
# TODO: look at sprites, styles, etc
|
||||
self.add_gui_object_include(sub_obj, fp)
|
||||
for sub_obj in repeat.findall('objects'):
|
||||
# TODO: look at sprites, styles, etc
|
||||
self.add_gui_object_include(sub_obj, fp)
|
||||
|
||||
self.add_gui_object_include(repeat, fp)
|
||||
|
||||
def add_gui_object_include(self, obj, fp):
|
||||
for include in obj.findall('include'):
|
||||
included_file = include.get('file')
|
||||
if included_file:
|
||||
self.deps.append((str(fp), f'{included_file}'))
|
||||
|
||||
def add_gui_object(self, parent, fp):
|
||||
if parent is None:
|
||||
return
|
||||
|
||||
for obj in parent.findall('object'):
|
||||
# TODO: look at sprites, styles, etc
|
||||
self.add_gui_object_repeat(obj, fp)
|
||||
self.add_gui_object_include(obj, fp)
|
||||
self.add_gui_object(obj, fp)
|
||||
for obj in parent.findall('objects'):
|
||||
# TODO: look at sprites, styles, etc
|
||||
self.add_gui_object_repeat(obj, fp)
|
||||
self.add_gui_object_include(obj, fp)
|
||||
self.add_gui_object(obj, fp)
|
||||
|
||||
|
||||
def add_gui_xml(self):
|
||||
self.logger.info("Loading GUI XML...")
|
||||
for (fp, ffp) in sorted(self.find_files('gui', 'xml')):
|
||||
self.files.append(str(fp))
|
||||
# GUI page definitions are assumed to be named page_[something].xml and alone in that.
|
||||
if match(r".*[\\\/]page(_[^.\/\\]+)?\.xml$", str(fp)):
|
||||
self.roots.append(str(fp))
|
||||
root_xml = ElementTree.parse(ffp).getroot()
|
||||
for include in root_xml.findall('include'):
|
||||
# If including an entire directory, find all the *.xml files
|
||||
if include.text.endswith('/'):
|
||||
self.deps.extend([(str(fp), str(sub_fp)) for (sub_fp, sub_ffp) in self.find_files(f'gui/{include.text}', 'xml')])
|
||||
else:
|
||||
self.deps.append((str(fp), f'gui/{include.text}'))
|
||||
else:
|
||||
xml = ElementTree.parse(ffp)
|
||||
root_xml = xml.getroot()
|
||||
name = root_xml.tag
|
||||
self.roots.append(str(fp))
|
||||
if name in ('objects', 'object'):
|
||||
for script in root_xml.findall('script'):
|
||||
if script.get('file'):
|
||||
self.deps.append((str(fp), script.get('file')))
|
||||
if script.get('directory'):
|
||||
# If including an entire directory, find all the *.js files
|
||||
self.deps.extend([(str(fp), str(sub_fp)) for (sub_fp, sub_ffp) in self.find_files(script.get('directory'), 'js')])
|
||||
self.add_gui_object(root_xml, fp)
|
||||
elif name == 'setup':
|
||||
# TODO: look at sprites, styles, etc
|
||||
pass
|
||||
elif name == 'styles':
|
||||
for style in root_xml.findall('style'):
|
||||
if(style.get('sound_opened')):
|
||||
self.deps.append((str(fp), f"{style.get('sound_opened')}"))
|
||||
if(style.get('sound_closed')):
|
||||
self.deps.append((str(fp), f"{style.get('sound_closed')}"))
|
||||
if(style.get('sound_selected')):
|
||||
self.deps.append((str(fp), f"{style.get('sound_selected')}"))
|
||||
if(style.get('sound_disabled')):
|
||||
self.deps.append((str(fp), f"{style.get('sound_disabled')}"))
|
||||
# TODO: look at sprites, styles, etc
|
||||
pass
|
||||
elif name == 'sprites':
|
||||
for sprite in root_xml.findall('sprite'):
|
||||
for image in sprite.findall('image'):
|
||||
if image.get('texture'):
|
||||
self.deps.append((str(fp), f"art/textures/ui/{image.get('texture')}"))
|
||||
else:
|
||||
bio = BytesIO()
|
||||
xml.write(bio)
|
||||
bio.seek(0)
|
||||
raise ValueError(f"Unexpected GUI XML root element '{name}':\n{bio.read().decode('ascii')}")
|
||||
|
||||
def add_gui_data(self):
|
||||
self.logger.info("Loading GUI data...")
|
||||
self.files.extend([str(fp) for (fp, ffp) in self.find_files('gui', 'js')])
|
||||
self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/ui', *self.supportedTextureFormats)])
|
||||
self.files.extend([str(fp) for (fp, ffp) in self.find_files('art/textures/selection', *self.supportedTextureFormats)])
|
||||
|
||||
def add_civs(self):
|
||||
self.logger.info("Loading civs...")
|
||||
for (fp, ffp) in sorted(self.find_files('simulation/data/civs', 'json')):
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
with open(ffp, encoding='utf-8') as f:
|
||||
civ = load(f)
|
||||
for music in civ.get('Music', []):
|
||||
self.deps.append((str(fp), f"audio/music/{music['File']}"))
|
||||
|
||||
def add_tips(self):
|
||||
self.logger.info("Loading tips...")
|
||||
for (fp, ffp) in sorted(self.find_files('gui/text/tips', 'txt')):
|
||||
relative_path = str(fp)
|
||||
self.files.append(relative_path)
|
||||
self.roots.append(relative_path)
|
||||
self.deps.append((relative_path, f"art/textures/ui/loading/tips/{basename(relative_path).split('.')[0]}.png"))
|
||||
|
||||
|
||||
def add_rms(self):
|
||||
self.logger.info("Loading random maps...")
|
||||
self.files.extend([str(fp) for (fp, ffp) in self.find_files('maps/random', 'js')])
|
||||
for (fp, ffp) in sorted(self.find_files('maps/random', 'json')):
|
||||
if str(fp).startswith('maps/random/rmbiome'):
|
||||
continue
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
with open(ffp, encoding='utf-8') as f:
|
||||
randmap = load(f)
|
||||
settings = randmap.get('settings', {})
|
||||
if settings.get('Script', None):
|
||||
self.deps.append((str(fp), f"maps/random/{settings['Script']}"))
|
||||
# Map previews
|
||||
if settings.get('Preview', None):
|
||||
self.deps.append((str(fp), f'art/textures/ui/session/icons/mappreview/{settings["Preview"]}'))
|
||||
|
||||
def add_techs(self):
|
||||
self.logger.info("Loading techs...")
|
||||
for (fp, ffp) in sorted(self.find_files('simulation/data/technologies', 'json')):
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
with open(ffp, encoding='utf-8') as f:
|
||||
tech = load(f)
|
||||
if tech.get('icon', None):
|
||||
self.deps.append((str(fp), f"art/textures/ui/session/portraits/technologies/{tech['icon']}"))
|
||||
if tech.get('supersedes', None):
|
||||
self.deps.append((str(fp), f"simulation/data/technologies/{tech['supersedes']}.json"))
|
||||
|
||||
def add_terrains(self):
|
||||
self.logger.info("Loading terrains...")
|
||||
for (fp, ffp) in sorted(self.find_files('art/terrains', 'xml')):
|
||||
# ignore terrains.xml
|
||||
if str(fp).endswith('terrains.xml'):
|
||||
continue
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
terrain = ElementTree.parse(ffp).getroot()
|
||||
for texture in terrain.find('textures').findall('texture'):
|
||||
if texture.get('file'):
|
||||
self.deps.append((str(fp), f"art/textures/terrain/{texture.get('file')}"))
|
||||
if terrain.find('material') is not None:
|
||||
material = terrain.find('material').text
|
||||
self.deps.append((str(fp), f"art/materials/{material}"))
|
||||
|
||||
def add_auras(self):
|
||||
self.logger.info("Loading auras...")
|
||||
for (fp, ffp) in sorted(self.find_files('simulation/data/auras', 'json')):
|
||||
self.files.append(str(fp))
|
||||
self.roots.append(str(fp))
|
||||
with open(ffp, encoding='utf-8') as f:
|
||||
aura = load(f)
|
||||
if aura.get('overlayIcon', None):
|
||||
self.deps.append((str(fp), aura['overlayIcon']))
|
||||
range_overlay = aura.get('rangeOverlay', {})
|
||||
for prop in ('lineTexture', 'lineTextureMask'):
|
||||
if range_overlay.get(prop, None):
|
||||
self.deps.append((str(fp), f"art/textures/selection/{range_overlay[prop]}"))
|
||||
|
||||
def check_deps(self):
|
||||
self.logger.info("Looking for missing files...")
|
||||
uniq_files = set(self.files)
|
||||
uniq_files = [r.replace(sep, '/') for r in uniq_files]
|
||||
lower_case_files = {f.lower(): f for f in uniq_files}
|
||||
reverse_deps = dict()
|
||||
for parent, dep in self.deps:
|
||||
if sep != '/':
|
||||
parent = parent.replace(sep, '/')
|
||||
dep = dep.replace(sep, '/')
|
||||
if dep not in reverse_deps:
|
||||
reverse_deps[dep] = {parent}
|
||||
else:
|
||||
reverse_deps[dep].add(parent)
|
||||
|
||||
for dep in sorted(reverse_deps.keys()):
|
||||
if "simulation/templates" in dep and (
|
||||
dep.replace("templates/", "template/special/filter/") in uniq_files or
|
||||
dep.replace("templates/", "template/mixins/") in uniq_files
|
||||
):
|
||||
continue
|
||||
|
||||
if dep in uniq_files:
|
||||
continue
|
||||
|
||||
callers = [str(self.vfs_to_relative_to_mods(ref)) for ref in reverse_deps[dep]]
|
||||
self.logger.warning(f"Missing file '{dep}' referenced by: {', '.join(sorted(callers))}")
|
||||
if dep.lower() in lower_case_files:
|
||||
self.logger.warning(f"### Case-insensitive match (found '{lower_case_files[dep.lower()]}')")
|
||||
|
||||
def check_unused(self):
|
||||
self.logger.info("Looking for unused files...")
|
||||
deps = dict()
|
||||
for parent, dep in self.deps:
|
||||
if sep != '/':
|
||||
parent = parent.replace(sep, '/')
|
||||
dep = dep.replace(sep, '/')
|
||||
|
||||
if parent not in deps:
|
||||
deps[parent] = {dep}
|
||||
else:
|
||||
deps[parent].add(dep)
|
||||
|
||||
uniq_files = set(self.files)
|
||||
uniq_files = [r.replace(sep, '/') for r in uniq_files]
|
||||
reachable = list(set(self.roots))
|
||||
reachable = [r.replace(sep, '/') for r in reachable]
|
||||
while True:
|
||||
new_reachable = []
|
||||
for r in reachable:
|
||||
new_reachable.extend([x for x in deps.get(r, {}) if x not in reachable])
|
||||
if new_reachable:
|
||||
reachable.extend(new_reachable)
|
||||
else:
|
||||
break
|
||||
|
||||
for f in sorted(uniq_files):
|
||||
if any((
|
||||
f in reachable,
|
||||
'art/terrains/' in f,
|
||||
'maps/random/' in f,
|
||||
)):
|
||||
continue
|
||||
self.logger.warning(f"Unused file '{str(self.vfs_to_relative_to_mods(f))}'")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
check_ref = CheckRefs()
|
||||
check_ref.main()
|
||||
48
source/tools/entity/creationgraph.py
Normal file
48
source/tools/entity/creationgraph.py
Normal file
@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
from os import chdir
|
||||
from pathlib import Path
|
||||
from re import split
|
||||
from subprocess import run
|
||||
from sys import exit
|
||||
from scriptlib import warn, SimulTemplateEntity, find_files
|
||||
|
||||
|
||||
def find_entities(vfs_root):
|
||||
base = vfs_root / 'public' / 'simulation' / 'templates'
|
||||
return [str(fp.relative_to(base).with_suffix('')) for (_, fp) in find_files(vfs_root, ['public'], 'simulation/templates', 'xml')]
|
||||
|
||||
|
||||
def main():
|
||||
vfs_root = Path(__file__).resolve().parents[3] / 'binaries' / 'data' / 'mods'
|
||||
simul_templates_path = Path('simulation/templates')
|
||||
simul_template_entity = SimulTemplateEntity(vfs_root)
|
||||
with open('creation.dot', 'w') as dot_f:
|
||||
dot_f.write('digraph G {\n')
|
||||
files = sorted(find_entities(vfs_root))
|
||||
for f in files:
|
||||
if f.startswith('template_'):
|
||||
continue
|
||||
print(f"# {f}...")
|
||||
entity = simul_template_entity.load_inherited(simul_templates_path, f, ['public'])
|
||||
if entity.find('Builder') is not None and entity.find('Builder').find('Entities') is not None:
|
||||
entities = entity.find('Builder').find('Entities').text.replace('{civ}', entity.find('Identity').find('Civ').text)
|
||||
builders = split(r'\s+', entities.strip())
|
||||
for builder in builders:
|
||||
if Path(builder) in files:
|
||||
warn(f"Invalid Builder reference: {f} -> {builder}")
|
||||
dot_f.write(f'"{f}" -> "{builder}" [color=green];\n')
|
||||
if entity.find('TrainingQueue') is not None and entity.find('TrainingQueue').find('Entities') is not None:
|
||||
entities = entity.find('TrainingQueue').find('Entities').text.replace('{civ}', entity.find('Identity').find('Civ').text)
|
||||
training_queues = split(r'\s+', entities.strip())
|
||||
for training_queue in training_queues:
|
||||
if Path(training_queue) in files:
|
||||
warn(f"Invalid TrainingQueue reference: {f} -> {training_queue}")
|
||||
dot_f.write(f'"{f}" -> "{training_queue}" [color=blue];\n')
|
||||
dot_f.write('}\n')
|
||||
if run(['dot', '-V'], capture_output=True).returncode == 0:
|
||||
exit(run(['dot', '-Tpng', 'creation.dot', '-o', 'creation.png'], text=True).returncode)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
chdir(Path(__file__).resolve().parent)
|
||||
main()
|
||||
44
source/tools/entity/entvalidate.py
Normal file
44
source/tools/entity/entvalidate.py
Normal file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
from os import chdir
|
||||
from pathlib import Path
|
||||
from subprocess import run, CalledProcessError
|
||||
from sys import exit
|
||||
from xml.etree import ElementTree
|
||||
from scriptlib import warn, SimulTemplateEntity, find_files
|
||||
|
||||
|
||||
def main():
|
||||
root = Path(__file__).resolve().parents[3]
|
||||
relaxng_schema = root / 'binaries' / 'system' / 'entity.rng'
|
||||
if not relaxng_schema.exists():
|
||||
warn(f"""Relax NG schema non existant.
|
||||
Please create the file {relaxng_schema.relative_to(root)}
|
||||
You can do that by running 'pyrogenesis -dumpSchema' in the 'system' directory""")
|
||||
exit(1)
|
||||
if run(['xmllint', '--version'], capture_output=True).returncode != 0:
|
||||
warn("xmllint not found in your PATH, please install it (usually in libxml2 package)")
|
||||
exit(2)
|
||||
vfs_root = root / 'binaries' / 'data' / 'mods'
|
||||
simul_templates_path = Path('simulation/templates')
|
||||
simul_template_entity = SimulTemplateEntity(vfs_root)
|
||||
count = 0
|
||||
failed = 0
|
||||
for fp, _ in sorted(find_files(vfs_root, ['public'], 'simulation/templates', 'xml')):
|
||||
if fp.stem.startswith('template_'):
|
||||
continue
|
||||
print(f"# {fp}...")
|
||||
count += 1
|
||||
entity = simul_template_entity.load_inherited(simul_templates_path, str(fp.relative_to(simul_templates_path)), ['public'])
|
||||
xmlcontent = ElementTree.tostring(entity, encoding='unicode')
|
||||
try:
|
||||
run(['xmllint', '--relaxng', str(relaxng_schema.resolve()), '-'], input=xmlcontent, capture_output=True, text=True, check=True)
|
||||
except CalledProcessError as e:
|
||||
failed += 1
|
||||
print(e.stderr)
|
||||
print(e.stdout)
|
||||
print(f"\nTotal: {count}; failed: {failed}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
chdir(Path(__file__).resolve().parent)
|
||||
main()
|
||||
@ -1,4 +1,4 @@
|
||||
# Checkrefs.pl
|
||||
# Checkrefs.py
|
||||
|
||||
## Description
|
||||
|
||||
@ -6,24 +6,27 @@ This script checks the game files for missing dependencies, unused files, and fo
|
||||
|
||||
## Requirements
|
||||
|
||||
- Perl interpreter installed
|
||||
- Dependencies:
|
||||
- XML::Parser
|
||||
- XML::Simple
|
||||
- Getopt::Long
|
||||
- File::Find
|
||||
- Data::Dumper
|
||||
- JSON
|
||||
- Python 3.6+ interpreter installed
|
||||
- lxml for the -a option.
|
||||
|
||||
## Usage
|
||||
|
||||
- cd in source/tools/entity and run the script.
|
||||
- cd in `source/tools/entity` and run the script.
|
||||
|
||||
```
|
||||
Usage: perl checkrefs.pl [OPTION]...
|
||||
usage: checkrefs.py [-h] [-u] [-x] [-a] [-t] [-m MOD [MOD ...]]
|
||||
|
||||
Checks the game files for missing dependencies, unused files, and for file integrity.
|
||||
--check-unused check for all the unused files in the given mods and their dependencies. Implies --check-map-xml. Currently yields a lot of false positives.
|
||||
--check-map-xml check maps for missing actor and templates.
|
||||
--validate-templates run the validate.pl script to check if the xml files match their (.rng) grammar file. This currently only works for the public mod.
|
||||
--mod-to-check=mods specify which mods to check. 'mods' should be a list of mods separated by '|'. Default value: 'public|mod'.
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-u, --check-unused check for all the unused files in the given mods and their dependencies. Implies --check-map-
|
||||
xml. Currently yields a lot of false positives.
|
||||
-x, --check-map-xml check maps for missing actor and templates.
|
||||
-a, --validate-actors
|
||||
run the validator.py script to check if the actors files have extra or missing textures.
|
||||
-t, --validate-templates
|
||||
run the validator.py script to check if the xml files match their (.rng) grammar file.
|
||||
-m MOD [MOD ...], --mods MOD [MOD ...]
|
||||
specify which mods to check. Default to public.
|
||||
```
|
||||
|
||||
128
source/tools/entity/scriptlib/__init__.py
Normal file
128
source/tools/entity/scriptlib/__init__.py
Normal file
@ -0,0 +1,128 @@
|
||||
from collections import Counter
|
||||
from decimal import Decimal
|
||||
from re import split
|
||||
from sys import stderr
|
||||
from xml.etree import ElementTree
|
||||
from os.path import exists
|
||||
|
||||
class SimulTemplateEntity:
|
||||
def __init__(self, vfs_root, logger):
|
||||
self.vfs_root = vfs_root
|
||||
self.logger = logger
|
||||
|
||||
def get_file(self, base_path, vfs_path, mod):
|
||||
default_path = self.vfs_root / mod / base_path
|
||||
file = (default_path/ "special" / "filter" / vfs_path).with_suffix('.xml')
|
||||
if not exists(file):
|
||||
file = (default_path / "mixins" / vfs_path).with_suffix('.xml')
|
||||
if not exists(file):
|
||||
file = (default_path / vfs_path).with_suffix('.xml')
|
||||
return file
|
||||
|
||||
def get_main_mod(self, base_path, vfs_path, mods):
|
||||
for mod in mods:
|
||||
fp = self.get_file(base_path, vfs_path, mod)
|
||||
if fp.exists():
|
||||
main_mod = mod
|
||||
break
|
||||
else:
|
||||
# default to first mod
|
||||
# it should then not exist
|
||||
# it will raise an exception when trying to read it
|
||||
main_mod = mods[0]
|
||||
return main_mod
|
||||
|
||||
def apply_layer(self, base_tag, tag):
|
||||
"""
|
||||
apply tag layer to base_tag
|
||||
"""
|
||||
if tag.get('datatype') == 'tokens':
|
||||
base_tokens = split(r'\s+', base_tag.text or '')
|
||||
tokens = split(r'\s+', tag.text or '')
|
||||
final_tokens = base_tokens.copy()
|
||||
for token in tokens:
|
||||
if token.startswith('-'):
|
||||
token_to_remove = token[1:]
|
||||
if token_to_remove in final_tokens:
|
||||
final_tokens.remove(token_to_remove)
|
||||
elif token not in final_tokens:
|
||||
final_tokens.append(token)
|
||||
base_tag.text = ' '.join(final_tokens)
|
||||
elif tag.get('op'):
|
||||
op = tag.get('op')
|
||||
op1 = Decimal(base_tag.text or '0')
|
||||
op2 = Decimal(tag.text or '0')
|
||||
if op == 'add':
|
||||
base_tag.text = str(op1 + op2)
|
||||
elif op == 'mul':
|
||||
base_tag.text = str(op1 * op2)
|
||||
elif op == 'mul_round':
|
||||
base_tag.text = str(round(op1 * op2))
|
||||
else:
|
||||
raise ValueError(f"Invalid operator '{op}'")
|
||||
else:
|
||||
base_tag.text = tag.text
|
||||
for child in tag:
|
||||
base_child = base_tag.find(child.tag)
|
||||
if 'disable' in child.attrib:
|
||||
if base_child is not None:
|
||||
base_tag.remove(base_child)
|
||||
else:
|
||||
if 'replace' in child.attrib and base_child is not None:
|
||||
base_tag.remove(base_child)
|
||||
if base_child is None:
|
||||
base_child = ElementTree.Element(child.tag)
|
||||
base_tag.append(base_child)
|
||||
self.apply_layer(base_child, child)
|
||||
if 'replace' in base_child.attrib:
|
||||
del base_child.attrib['replace']
|
||||
|
||||
def load_inherited(self, base_path, vfs_path, mods, base = None):
|
||||
"""
|
||||
vfs_path should be relative to base_path in a mod
|
||||
"""
|
||||
if '|' in vfs_path:
|
||||
paths = vfs_path.split("|", 2)
|
||||
base = self.load_inherited(base_path, paths[1], mods, base);
|
||||
base = self.load_inherited(base_path, paths[0], mods, base);
|
||||
return base
|
||||
|
||||
main_mod = self.get_main_mod(base_path, vfs_path, mods)
|
||||
fp = self.get_file(base_path, vfs_path, main_mod)
|
||||
layer = ElementTree.parse(fp).getroot()
|
||||
for el in layer.iter():
|
||||
children = [x.tag for x in el]
|
||||
duplicates = [x for x, c in Counter(children).items() if c > 1]
|
||||
if duplicates:
|
||||
for dup in duplicates:
|
||||
self.logger.warning(f"Duplicate child node '{dup}' in tag {el.tag} of {fp}")
|
||||
if layer.get('parent'):
|
||||
parent = self.load_inherited(base_path, layer.get('parent'), mods)
|
||||
self.apply_layer(parent, layer)
|
||||
return parent
|
||||
else:
|
||||
if not base:
|
||||
return layer
|
||||
else:
|
||||
self.apply_layer(base, layer)
|
||||
return base
|
||||
|
||||
|
||||
def find_files(vfs_root, mods, vfs_path, *ext_list):
|
||||
"""
|
||||
returns a list of 2-size tuple with:
|
||||
- Path relative to the mod base
|
||||
- full Path
|
||||
"""
|
||||
full_exts = ['.' + ext for ext in ext_list]
|
||||
|
||||
def find_recursive(dp, base):
|
||||
"""(relative Path, full Path) generator"""
|
||||
if dp.is_dir():
|
||||
if dp.name != '.svn' and dp.name != '.git' and not dp.name.endswith('~'):
|
||||
for fp in dp.iterdir():
|
||||
yield from find_recursive(fp, base)
|
||||
elif dp.suffix in full_exts:
|
||||
relative_file_path = dp.relative_to(base)
|
||||
yield (relative_file_path, dp.resolve())
|
||||
return [(rp, fp) for mod in mods for (rp, fp) in find_recursive(vfs_root / mod / vfs_path, vfs_root / mod)]
|
||||
@ -4,8 +4,20 @@ from pathlib import Path
|
||||
from os.path import sep, join, realpath, exists, basename, dirname
|
||||
from json import load, loads
|
||||
from re import split, match
|
||||
from logging import getLogger, StreamHandler, INFO, Formatter
|
||||
from logging import getLogger, StreamHandler, INFO, WARNING, Filter, Formatter
|
||||
import lxml.etree
|
||||
import sys
|
||||
|
||||
class SingleLevelFilter(Filter):
|
||||
def __init__(self, passlevel, reject):
|
||||
self.passlevel = passlevel
|
||||
self.reject = reject
|
||||
|
||||
def filter(self, record):
|
||||
if self.reject:
|
||||
return (record.levelno != self.passlevel)
|
||||
else:
|
||||
return (record.levelno == self.passlevel)
|
||||
|
||||
class VFS_File:
|
||||
def __init__(self, mod_name, vfs_path):
|
||||
@ -24,11 +36,16 @@ class RelaxNGValidator:
|
||||
logger = getLogger(__name__)
|
||||
logger.setLevel(INFO)
|
||||
# create a console handler, seems nicer to Windows and for future uses
|
||||
ch = StreamHandler()
|
||||
ch = StreamHandler(sys.stdout)
|
||||
ch.setLevel(INFO)
|
||||
ch.setFormatter(Formatter('%(levelname)s - %(message)s'))
|
||||
# ch.setFormatter(Formatter('%(message)s')) # same output as perl
|
||||
f1 = SingleLevelFilter(INFO, False)
|
||||
ch.addFilter(f1)
|
||||
logger.addHandler(ch)
|
||||
errorch = StreamHandler(sys.stderr)
|
||||
errorch.setLevel(WARNING)
|
||||
errorch.setFormatter(Formatter('%(levelname)s - %(message)s'))
|
||||
logger.addHandler(errorch)
|
||||
self.logger = logger
|
||||
|
||||
def run (self):
|
||||
@ -1,10 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
import time
|
||||
import xml.etree.ElementTree
|
||||
from logging import getLogger, StreamHandler, INFO, Formatter
|
||||
from logging import getLogger, StreamHandler, INFO, WARNING, Formatter, Filter
|
||||
|
||||
class SingleLevelFilter(Filter):
|
||||
def __init__(self, passlevel, reject):
|
||||
self.passlevel = passlevel
|
||||
self.reject = reject
|
||||
|
||||
def filter(self, record):
|
||||
if self.reject:
|
||||
return (record.levelno != self.passlevel)
|
||||
else:
|
||||
return (record.levelno == self.passlevel)
|
||||
|
||||
class Actor:
|
||||
def __init__(self, mod_name, vfs_path):
|
||||
@ -19,7 +30,7 @@ class Actor:
|
||||
try:
|
||||
tree = xml.etree.ElementTree.parse(physical_path)
|
||||
except xml.etree.ElementTree.ParseError as err:
|
||||
self.logger.error('"%s": %s\n' % (physical_path, err.msg))
|
||||
self.logger.error('"%s": %s' % (physical_path, err.msg))
|
||||
return False
|
||||
root = tree.getroot()
|
||||
# Special case: particles don't need a diffuse texture.
|
||||
@ -41,7 +52,7 @@ class Actor:
|
||||
try:
|
||||
tree = xml.etree.ElementTree.parse(physical_path)
|
||||
except xml.etree.ElementTree.ParseError as err:
|
||||
self.logger.error('"%s": %s\n' % (physical_path, err.msg))
|
||||
self.logger.error('"%s": %s' % (physical_path, err.msg))
|
||||
return False
|
||||
|
||||
root = tree.getroot()
|
||||
@ -64,7 +75,7 @@ class Material:
|
||||
try:
|
||||
root = xml.etree.ElementTree.parse(physical_path).getroot()
|
||||
except xml.etree.ElementTree.ParseError as err:
|
||||
self.logger.error('"%s": %s\n' % (physical_path, err.msg))
|
||||
self.logger.error('"%s": %s' % (physical_path, err.msg))
|
||||
return False
|
||||
for element in root.findall('.//required_texture'):
|
||||
texture_name = element.get('name')
|
||||
@ -88,10 +99,17 @@ class Validator:
|
||||
def __init_logger(self):
|
||||
logger = getLogger(__name__)
|
||||
logger.setLevel(INFO)
|
||||
ch = StreamHandler()
|
||||
# create a console handler, seems nicer to Windows and for future uses
|
||||
ch = StreamHandler(sys.stdout)
|
||||
ch.setLevel(INFO)
|
||||
ch.setFormatter(Formatter('%(levelname)s - %(message)s'))
|
||||
f1 = SingleLevelFilter(INFO, False)
|
||||
ch.addFilter(f1)
|
||||
logger.addHandler(ch)
|
||||
errorch = StreamHandler(sys.stderr)
|
||||
errorch.setLevel(WARNING)
|
||||
errorch.setFormatter(Formatter('%(levelname)s - %(message)s'))
|
||||
logger.addHandler(errorch)
|
||||
self.logger = logger
|
||||
|
||||
def get_mod_path(self, mod_name, vfs_path):
|
||||
@ -126,6 +144,7 @@ class Validator:
|
||||
return result
|
||||
|
||||
def find_materials(self, vfs_path):
|
||||
self.logger.info('Collecting materials...')
|
||||
material_files = self.find_all_mods_files(vfs_path, re.compile(r'.*\.xml'))
|
||||
for material_file in material_files:
|
||||
material_name = os.path.basename(material_file['vfs_path'])
|
||||
@ -138,6 +157,8 @@ class Validator:
|
||||
self.invalid_materials[material_name] = material
|
||||
|
||||
def find_actors(self, vfs_path):
|
||||
self.logger.info('Collecting actors...')
|
||||
|
||||
actor_files = self.find_all_mods_files(vfs_path, re.compile(r'.*\.xml'))
|
||||
for actor_file in actor_files:
|
||||
actor = Actor(actor_file['mod_name'], actor_file['vfs_path'])
|
||||
@ -145,12 +166,9 @@ class Validator:
|
||||
self.actors.append(actor)
|
||||
|
||||
def run(self):
|
||||
start_time = time.time()
|
||||
|
||||
self.logger.info('Collecting list of files to check\n')
|
||||
|
||||
self.find_materials(os.path.join('art', 'materials'))
|
||||
self.find_actors(os.path.join('art', 'actors'))
|
||||
self.logger.info('Validating textures...')
|
||||
|
||||
for actor in self.actors:
|
||||
if not actor.material:
|
||||
@ -180,9 +198,6 @@ class Validator:
|
||||
material.name
|
||||
))
|
||||
|
||||
finish_time = time.time()
|
||||
self.logger.info('Total execution time: %.3f seconds.\n' % (finish_time - start_time))
|
||||
|
||||
if __name__ == '__main__':
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
default_root = os.path.join(script_dir, '..', '..', '..', 'binaries', 'data', 'mods')
|
||||
|
||||
Loading…
Reference in New Issue
Block a user