# Added some tools to automatically build the game from the source code

Various scripts and stuff that run on the build server

This was SVN commit r6571.

source/tools/autobuild2/aws.conf.example

@@ -0,0 +1,9 @@
+aws_access_key_id: XXXXXXXXXXXXXXXXXXXX
+aws_secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+ec2_metadata_root: http://169.254.169.254/2007-08-29/meta-data/
+ebs_volume_id: vol-XXXXXXXX
+ebs_device: xvdg
+ebs_drive_letter: e
+sync: c:\bin\sync.exe
+svn_username: autobuild
+svn_password: XXXXXXXXX

source/tools/autobuild2/build.pl

@@ -0,0 +1,128 @@
+# Build script - does the actual building of the project
+
+use strict;
+use warnings;
+
+use constant EXIT_BUILDCOMPLETE => 0;
+use constant EXIT_FAILED => 1;
+
+my %config = load_conf("c:\\0ad\\autobuild\\aws.conf");
+
+my $svn_trunk = "e:\\svn";
+my $temp_trunk = "d:\\0ad";
+my $log_dir = "d:\\0ad\\buildlogs";
+my $vcbuild = "$svn_trunk\\source\\tools\\autobuild2\\vcbuild_env.bat";
+
+my $time_start = time();
+
+eval { # catch deaths
+
+# Clean the output directory, just in case it's got old junk left over somehow
+`rmdir /q /s $temp_trunk 2>&1`; # (ignore failures, they don't matter)
+mkdir $temp_trunk or die $!;
+mkdir $log_dir or die $!;
+
+# Capture all output
+open STDOUT, '>', "$log_dir\\build_stdout.txt";
+open STDERR, '>', "$log_dir\\build_stderr.txt";
+open BUILDLOG, '>', "$log_dir\\buildlog.txt" or die $!;
+
+add_to_buildlog("Starting build");
+
+chdir $svn_trunk or die $!;
+
+# Copy all the necessary files into the temporary working area
+
+# For some directories, do a real copy
+for (qw(build source libraries))
+{
+    add_to_buildlog("xcopying $_");
+    `xcopy /e $svn_trunk\\$_ $temp_trunk\\$_\\ 2>&1`;
+    die "xcopy $_: $?" if $?;
+}
+
+# # For other directories (which we're only going to read), do a 'junction' (like a symbolic link) because it's faster
+# #
+# # Actually don't, because it's easier to not have to install the junction tool
+# for (qw(source libraries))
+# {
+#     `$junction $temp_trunk\\$_ $svn_trunk\\$_`;
+#     die "junction $_: $?" if $?;
+# }
+
+# Create the workspace files
+
+add_to_buildlog("Running update-workspaces");
+chdir "$temp_trunk\\build\\workspaces" or die $!;
+my $updateworkspaces_output = `update-workspaces.bat 2>&1`;
+add_to_buildlog($updateworkspaces_output);
+die $? if $?;
+
+# Create target directories for built files
+
+mkdir "$temp_trunk\\binaries" or die $!;
+mkdir "$temp_trunk\\binaries\\system" or die $!;
+
+# Do the Release build
+
+add_to_buildlog("Running vcbuild");
+my $build_output = `$vcbuild /time /M2 /logfile:$log_dir\\build_vcbuild.txt vc2008\\pyrogenesis.sln "Release|Win32" 2>&1`;
+add_to_buildlog($build_output);
+die $? if ($? and $? != 32768);
+# { open my $f, '>', "$temp_trunk/binaries/system/pyrogenesis.exe"; print $f "exe\n" }
+# { open my $f, '>', "$temp_trunk/binaries/system/pyrogenesis.pdb"; print $f "pdb\n" }
+
+# Copy the output
+
+add_to_buildlog("Copying generated binaries");
+`copy $temp_trunk\\binaries\\system\\pyrogenesis.exe $svn_trunk\\binaries\\system\\`;
+die $? if $?;
+`copy $temp_trunk\\binaries\\system\\pyrogenesis.pdb $svn_trunk\\binaries\\system\\`;
+die $? if $?;
+
+# Commit to SVN
+
+my $svn_output = `svn commit --username $config{svn_username} --password $config{svn_password} $svn_trunk\\binaries\\system\\pyrogenesis.exe $svn_trunk\\binaries\\system\\pyrogenesis.pdb --message "Automated build." 2>&1`;
+add_to_buildlog($svn_output);
+die $? if $?;
+
+}; # end of eval
+
+if ($@)
+{
+    warn $@;
+    quit(EXIT_FAILED);
+}
+else
+{
+    quit(EXIT_BUILDCOMPLETE);
+}
+
+
+# Exit, after copying the current log files over the previous ones
+sub quit
+{
+    my $time_end = time();
+    my $time_taken = $time_end - $time_start;
+    add_to_buildlog("Build completed with code $_[0] - took $time_taken seconds.");
+
+    exit($_[0]);
+}
+
+
+sub add_to_buildlog
+{
+    print BUILDLOG +(gmtime time)."\n$_[0]\n--------------------------------------------------------------------------------\n";
+}
+
+sub load_conf {
+    my ($filename) = @_;
+    open my $f, '<', $filename or die "Failed to open $filename: $!";
+    my %c;
+    while (<$f>) {
+        if (/^(.+?): (.+)/) {
+            $c{$1} = $2;
+        }
+    }
+    return %c;
+}

source/tools/autobuild2/logindex.html

@@ -0,0 +1,41 @@
+<!DOCTYPE html>
+<title>WFG Autobuilder Logs</title>
+<style>
+body { font-family: sans-serif; font-size: small; }
+ul { margin: 0; padding: 0; }
+li { list-style: none; margin: 0; padding: 0; }
+</style>
+
+<ul id="logs">
+<li>Loading...
+</ul>
+
+<script>
+window.onload = function () {
+    var xhr = new XMLHttpRequest();
+    xhr.open('GET', '/');
+    xhr.onreadystatechange = function () {
+        if (xhr.readyState == 4) {
+            if (xhr.status != 200) {
+                document.getElementById('logs').innerHTML = '<li>HTTP error '+xhr.status;
+                return;
+            }
+
+            var items = [];
+            var contents = xhr.responseXML.getElementsByTagName('Contents');
+            for (var i = 0; i < contents.length; ++i) {
+                var key = contents[i].getElementsByTagName('Key')[0].textContent;
+                if (key == 'logindex.html') continue;
+                var size = contents[i].getElementsByTagName('Size')[0].textContent;
+                items.push( [key, size] );
+            }
+            items.sort(function (a, b) { return a[0] < b[0] ? 1 : a[0] > b[0] ? -1 : 0 });
+            var output = '';
+            for (var i = 0; i < items.length; ++i)
+                output += '<li><a href="' + escape(items[i][0]) + '">' + items[i][0] + '</a> (' + items[i][1] + ' bytes)';
+            document.getElementById('logs').innerHTML = output;
+        }
+    }
+    xhr.send('');
+}
+</script>

source/tools/autobuild2/startup.pl

@@ -0,0 +1,219 @@
+=pod
+
+This script is executed on startup (via a service) on the build server.
+
+It is responsible for:
+ * attaching the necessary disks,
+ * updating from SVN,
+ * executing the rest of the build script that's in SVN,
+ * cleaning up at the end,
+ * and saving the logs of everything that's going on.
+
+This script does as little as possible, since it is necessarily frozen
+into the static machine image and is hard to update.
+
+=cut
+
+use strict;
+use warnings;
+
+use Net::Amazon::EC2;
+use Amazon::S3;
+use LWP::Simple();
+use DateTime;
+
+my %config = load_conf("c:\\0ad\\autobuild\\aws.conf");
+my $timestamp = DateTime->now->iso8601;
+
+my $s3 = new Amazon::S3( {
+    aws_access_key_id => $config{aws_access_key_id},
+    aws_secret_access_key => $config{aws_secret_access_key},
+    retry => 1,
+} );
+
+
+my $bucket = $s3->bucket('wfg-autobuild-logs');
+
+my $log = '';
+
+$SIG{__WARN__} = sub {
+    write_log("Warning: @_");
+    warn @_;
+};
+
+write_log("Starting");
+flush_log();
+
+my $ec2;
+
+$SIG{__DIE__} = sub {
+    die @_ if $^S; # ignore deaths in eval
+
+    write_log("Died\n@_");
+    flush_log();
+    if ($ec2) {
+        terminate_instance();
+    }
+    die @_;
+};
+
+$ec2 = new Net::Amazon::EC2(
+    AWSAccessKeyId => $config{aws_access_key_id},
+    SecretAccessKey => $config{aws_secret_access_key},
+);
+
+my $instance_id = get_instance_id();
+write_log("Running on instance $instance_id");
+flush_log();
+
+attach_disk();
+
+update_svn();
+
+run_build_script();
+
+save_buildlogs();
+
+detach_disk();
+
+write_log("Finished");
+
+terminate_instance();
+
+exit;
+
+sub update_svn {
+    write_log("Updating from SVN");
+    my $output = `svn up --username $config{svn_username} --password $config{svn_password} e:\\svn 2>&1`;
+    write_log("svn up:\n================================\n$output\n================================\n");
+}
+
+sub run_build_script {
+    write_log("Running build script");
+    system("perl e:\\svn\\source\\tools\\autobuild2\\build.pl");
+}
+
+sub save_buildlogs {
+    opendir my $d, "d:\\0ad\\buildlogs" or die "Can't open buildlogs directory: $!";
+    for my $fn (sort readdir $d) {
+        next if $fn =~ /^\./;
+        open my $f, '<', "d:\\0ad\\buildlogs\\$fn" or die "Can't open buildlogs file $fn: $!";
+        my $data = do { local $/; <$f> };
+        write_log("$fn:\n================================\n$data\n================================\n");
+    }
+}
+
+sub attach_disk {
+    write_log("Attaching volume $config{ebs_volume_id} as $config{ebs_device}");
+    my $status = $ec2->attach_volume(
+        InstanceId => $instance_id,
+        VolumeId => $config{ebs_volume_id},
+        Device => $config{ebs_device},
+    );
+    write_log("Attached");
+
+    # Wait for the disk to get attached and visible
+    write_log("Waiting for volume to be visible");
+    my $mounts;
+    for my $i (0..60) {
+        # mountvol emits a list of volumes, so wait until the expected one is visible
+        $mounts = `mountvol`;
+        my $letter = uc $config{ebs_drive_letter};
+        if ($mounts =~ /(\\\\\?\\Volume\{\S+\}\\)\s+$letter:\\/) {
+            my $volume = $1;
+            write_log("Already got volume $volume mounted on drive $config{ebs_drive_letter}");
+            return;
+        } elsif ($mounts =~ /(\\\\\?\\Volume\{\S+\}\\)\s+\*\*\* NO MOUNT POINTS \*\*\*/) {
+            my $volume = $1;
+            write_log("Mounting volume $volume onto drive $config{ebs_drive_letter}");
+            system("mountvol $config{ebs_drive_letter}: $volume");
+            return;
+        }
+        write_log("Not seen the volume yet ($i)");
+        sleep 1;
+    }
+
+    die "Failed to find new volume. mountvol said:\n\n$mounts";
+}
+
+sub detach_disk {
+    # Based on http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1841&categoryID=174
+    write_log("Syncing drive $config{ebs_drive_letter}");
+    system("$config{sync} -r $config{ebs_drive_letter}:");
+
+    write_log("Unmounting drive $config{ebs_drive_letter}");
+    system("mountvol $config{ebs_drive_letter}: /d");
+
+    write_log("Detaching volume");
+    my $status = $ec2->detach_volume(
+        InstanceId => $instance_id,
+        VolumeId => $config{ebs_volume_id},
+    );
+    write_log("Detached");
+}
+
+sub terminate_instance {
+    write_log("Terminating instance (after a delay)");
+    flush_log();
+
+#     write_log("NOT REALLY");
+#     flush_log();
+#     return;
+
+    # Delay for a while, to give me a chance to log in and manually
+    # abort the termination if I want to configure the machine instead
+    # of having it die straightaway
+    sleep 60*5;
+    write_log("Really terminating now");
+    flush_log();
+    my $statuses = $ec2->terminate_instances(
+        InstanceId => $instance_id,
+    );
+    write_log("Terminated");
+    flush_log();
+}
+
+sub get_instance_id {
+    my $instance_id = LWP::Simple::get("$config{ec2_metadata_root}instance-id");
+    die "Invalid instance-id return value '$instance_id'" unless $instance_id =~ /\Ai-[a-f0-9]+\z/;
+    return $instance_id;
+}
+
+sub write_log {
+    my ($msg) = @_;
+
+    print "$msg\n";
+    my $t = scalar gmtime;
+    $log .= "$t: $msg\n\n";
+
+    # Instead of using the explicit flush_log, just flush all
+    # the time because it helps with debugging
+    flush_log_really();
+}
+
+sub flush_log_really {
+    my $filename = "$timestamp.startup";
+
+    my $ok = $bucket->add_key($filename, $log, {
+        acl_short => 'public-read',
+        'Content-Type' => 'text/plain',
+    });
+    warn "Failed - ".$bucket->errstr if not $ok;
+}
+
+sub flush_log {
+#     flush_log_really();
+}
+
+
+sub load_conf {
+    my ($filename) = @_;
+    open my $f, '<', $filename or die "Failed to open $filename: $!";
+    my %c;
+    while (<$f>) {
+        if (/^(.+?): (.+)/) {
+            $c{$1} = $2;
+        }
+    }
+    return %c;
+}

source/tools/autobuild2/vcbuild_env.bat

@@ -0,0 +1,2 @@
+call "C:\Program Files\Microsoft Visual Studio 9.0\VC\bin\vcvars32.bat"
+vcbuild %*