# Updated autobuilder
Also added the autobuild manager scripts This was SVN commit r6697.
This commit is contained in:
parent
394579ca7d
commit
6a7fdd8315
@ -1,9 +1,2 @@
|
||||
aws_access_key_id: XXXXXXXXXXXXXXXXXXXX
|
||||
aws_secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||
ec2_metadata_root: http://169.254.169.254/2007-08-29/meta-data/
|
||||
ebs_volume_id: vol-XXXXXXXX
|
||||
ebs_device: xvdg
|
||||
ebs_drive_letter: e
|
||||
sync: c:\bin\sync.exe
|
||||
svn_username: autobuild
|
||||
svn_password: XXXXXXXXX
|
||||
|
321
source/tools/autobuild2/manage.cgi
Executable file
321
source/tools/autobuild2/manage.cgi
Executable file
@ -0,0 +1,321 @@
|
||||
#!/usr/bin/perl -wT
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use CGI::Simple;
|
||||
use CGI::Carp qw(fatalsToBrowser);
|
||||
use DBI;
|
||||
use Net::Amazon::EC2;
|
||||
use DateTime::Format::ISO8601;
|
||||
use Archive::Zip;
|
||||
use MIME::Base64;
|
||||
use IO::String;
|
||||
use Data::Dumper;
|
||||
|
||||
my $root = '/var/svn/autobuild';
|
||||
|
||||
my %config = load_conf("$root/manage.conf");
|
||||
|
||||
my $cgi = new CGI::Simple;
|
||||
|
||||
my $user = $cgi->remote_user;
|
||||
die unless $user;
|
||||
|
||||
my $dbh = DBI->connect("dbi:SQLite:dbname=$root/$config{database}", '', '', { RaiseError => 1 });
|
||||
|
||||
my $ec2 = new Net::Amazon::EC2(
|
||||
AWSAccessKeyId => $config{aws_access_key_id},
|
||||
SecretAccessKey => $config{aws_secret_access_key},
|
||||
);
|
||||
|
||||
my $action = $cgi->url_param('action');
|
||||
|
||||
if (not defined $action or $action eq 'index') {
|
||||
log_action('index');
|
||||
print_index('');
|
||||
|
||||
} elsif ($action eq 'start') {
|
||||
die "Must be POST" unless $cgi->request_method eq 'POST';
|
||||
log_action('start');
|
||||
|
||||
=pod
|
||||
|
||||
Only one instance may run at once, and we need to prevent race conditions.
|
||||
So:
|
||||
* Use SQLite as a mutex, so only one CGI script can be trying to start at once
|
||||
* If the last attempted start was only a few seconds ago, reject this one since
|
||||
it's probably a double-click or something
|
||||
* Check the list of active machines. If it's non-empty, reject this request.
|
||||
* Otherwise, start the new machine.
|
||||
|
||||
=cut
|
||||
|
||||
$dbh->begin_work;
|
||||
die unless 1 == $dbh->do('UPDATE state SET value = ? WHERE key = ?', undef, $$, 'process_mutex');
|
||||
my ($last_start) = $dbh->selectrow_array('SELECT value FROM state WHERE key = ?', undef, 'last_start');
|
||||
my $last_start_age = time() - $last_start;
|
||||
die "Last start was only $last_start_age seconds ago - please try again later."
|
||||
if $last_start_age < 10;
|
||||
die unless 1 == $dbh->do('UPDATE state SET value = ? WHERE key = ?', undef, time(), 'last_start');
|
||||
|
||||
my $instances_status = get_ec2_status_table();
|
||||
for (@$instances_status) {
|
||||
if ($_->{instance_state} ne 'terminated') {
|
||||
die "Already got an active instance ($_->{instance_id}) - can't start another one.";
|
||||
}
|
||||
}
|
||||
|
||||
# No instances are currently active, and nobody else is in this
|
||||
# instance-starting script, so it's safe to start a new one
|
||||
|
||||
my $instances = start_ec2_instance();
|
||||
|
||||
$dbh->commit;
|
||||
|
||||
for (@$instances) {
|
||||
$dbh->do('INSERT INTO instances VALUES (?, ?)', undef, $_->{instance_id}, DateTime->now->iso8601);
|
||||
}
|
||||
|
||||
print_index(generate_status_table($instances, 'Newly started instance') . '<hr>');
|
||||
|
||||
} elsif ($action eq 'stop') {
|
||||
die "Must be POST" unless $cgi->request_method eq 'POST';
|
||||
my $id = $cgi->url_param('instance_id');
|
||||
$id =~ /\Ai-[0-9a-f]+\z/ or die "Invalid instance_id";
|
||||
log_action('stop', $id);
|
||||
stop_ec2_instance($id);
|
||||
print_index("<strong>Stopping instance $id</strong><hr>");
|
||||
|
||||
} elsif ($action eq 'console') {
|
||||
my $id = $cgi->url_param('instance_id');
|
||||
$id =~ /\Ai-[0-9a-f]+\z/ or die "Invalid instance_id";
|
||||
log_action('console', $id);
|
||||
my $output = get_console_output($id);
|
||||
$output =~ s/</</g;
|
||||
print_index("<strong>Console output from $id:</strong><pre>\n$output</pre><hr>");
|
||||
|
||||
} elsif ($action eq 'activity') {
|
||||
my $days = int $cgi->url_param('days') || 7;
|
||||
print_activity($days);
|
||||
|
||||
} else {
|
||||
log_action('invalid', $action);
|
||||
die "Invalid action '$action'";
|
||||
}
|
||||
|
||||
$dbh->disconnect;
|
||||
|
||||
sub print_index {
|
||||
my ($info) = @_;
|
||||
|
||||
my $instances_status = get_ec2_status_table();
|
||||
my $got_active_instance;
|
||||
for (@$instances_status) {
|
||||
if ($_->{instance_state} ne 'terminated') {
|
||||
$got_active_instance = $_->{instance_id} || '?';
|
||||
}
|
||||
}
|
||||
|
||||
my $status = generate_status_table($instances_status, 'Current EC2 machine status');
|
||||
print <<EOF;
|
||||
Pragma: no-cache
|
||||
Content-Type: text/html
|
||||
|
||||
<!DOCTYPE html>
|
||||
<title>0 A.D. autobuild manager</title>
|
||||
<link rel="stylesheet" href="manage.css">
|
||||
<p>Hello <i>$user</i>.</p>
|
||||
<p>
|
||||
<a href="?action=index">Refresh status</a> |
|
||||
<a href="http://wfg-autobuild-logs.s3.amazonaws.com/logindex.html">View build logs</a>
|
||||
</p>
|
||||
<hr>
|
||||
$info
|
||||
$status
|
||||
<p>
|
||||
EOF
|
||||
|
||||
if ($got_active_instance) {
|
||||
print qq{<button disabled title="Already running an instance ($got_active_instance)">Start new build</button>\n};
|
||||
} else {
|
||||
print qq{<form action="?action=start" method="post" onsubmit="return confirm('Are you sure you want to start a new build?')"><button type="submit">Start new build</button></form>\n};
|
||||
}
|
||||
}
|
||||
|
||||
sub log_action {
|
||||
my ($action, $params) = @_;
|
||||
$dbh->do('INSERT INTO activity (user, ip, ua, action, params) VALUES (?, ?, ?, ?, ?)',
|
||||
undef, $user, $cgi->remote_addr, $cgi->user_agent, $action, $params);
|
||||
}
|
||||
|
||||
sub print_activity {
|
||||
my ($days) = @_;
|
||||
print <<EOF;
|
||||
Content-Type: text/html
|
||||
|
||||
<!DOCTYPE html>
|
||||
<title>0 A.D. autobuild activity log</title>
|
||||
<link rel="stylesheet" href="manage.css">
|
||||
<table>
|
||||
<tr><th>Date<th>User<th>IP<th>Action<th>Params<th>UA
|
||||
EOF
|
||||
|
||||
my $sth = $dbh->prepare("SELECT * FROM activity WHERE timestamp > datetime('now', ?) ORDER BY id DESC");
|
||||
$sth->execute("-$days day");
|
||||
while (my $row = $sth->fetchrow_hashref) {
|
||||
print '<tr>';
|
||||
print '<td>'.$cgi->escapeHTML($row->{$_}) for qw(timestamp user ip action params ua);
|
||||
print "\n";
|
||||
}
|
||||
|
||||
print <<EOF;
|
||||
</table>
|
||||
EOF
|
||||
|
||||
}
|
||||
|
||||
sub generate_status_table {
|
||||
my ($instances, $caption) = @_;
|
||||
my @columns = (
|
||||
[ reservation_id => 'Reservation ID' ],
|
||||
[ instance_id => 'Instance ID' ],
|
||||
[ instance_state => 'State' ],
|
||||
[ image_id => 'Image ID '],
|
||||
[ dns_name => 'DNS name' ],
|
||||
[ launch_time => 'Launch time' ],
|
||||
[ reason => 'Last change' ],
|
||||
);
|
||||
my $count = @$instances;
|
||||
my $status = qq{<table id="status">\n<caption>$caption — $count instances</caption>\n<tr>};
|
||||
for (@columns) { $status .= qq{<th>$_->[1]}; }
|
||||
$status .= qq{\n};
|
||||
|
||||
for my $item (@$instances) {
|
||||
$status .= qq{<tr>};
|
||||
for (@columns) {
|
||||
if ($_->[0] eq 'launch_time') {
|
||||
my $t = DateTime::Format::ISO8601->parse_datetime($item->{$_->[0]});
|
||||
my $now = DateTime->now();
|
||||
my $diff = $now - $t;
|
||||
my ($hours, $minutes) = $diff->in_units('hours', 'minutes');
|
||||
my $age = "$minutes minutes ago";
|
||||
$age = "$hours hours, $age" if $hours;
|
||||
$status .= qq{<td>$item->{$_->[0]} ($age)};
|
||||
} else {
|
||||
$status .= qq{<td>$item->{$_->[0]}};
|
||||
}
|
||||
}
|
||||
$status .= qq{<td><a href="?action=console;instance_id=$item->{instance_id}">Console output</a>\n};
|
||||
$status .= qq{<td><form action="?action=stop;instance_id=$item->{instance_id}" method="post" onsubmit="return confirm('Are you sure you want to terminate this instance?')"><button type="submit">Terminate</button></form>\n};
|
||||
}
|
||||
|
||||
$status .= qq{</table>};
|
||||
return $status;
|
||||
}
|
||||
|
||||
sub flatten_instance {
|
||||
my ($reservation, $instance) = @_;
|
||||
return {
|
||||
reservation_id => $reservation->reservation_id,
|
||||
instance_id => $instance->instance_id,
|
||||
instance_state => $instance->instance_state->name,
|
||||
image_id => $instance->image_id,
|
||||
dns_name => $instance->dns_name,
|
||||
launch_time => $instance->launch_time,
|
||||
reason => $instance->reason,
|
||||
};
|
||||
}
|
||||
|
||||
sub get_ec2_status_table {
|
||||
# return [ ];
|
||||
# return [ {
|
||||
# reservation_id => 'r-12345678',
|
||||
# instance_id => 'i-12345678',
|
||||
# instance_state => 'pending',
|
||||
# image_id => 'ami-12345678',
|
||||
# dns_name => '',
|
||||
# launch_time => '2008-12-30T17:14:22.000Z',
|
||||
# reason => '',
|
||||
# } ];
|
||||
|
||||
my $reservations = $ec2->describe_instances();
|
||||
my @ret = ();
|
||||
for my $reservation (@$reservations) {
|
||||
push @ret, map flatten_instance($reservation, $_), @{$reservation->instances_set};
|
||||
}
|
||||
return \@ret;
|
||||
}
|
||||
|
||||
sub get_console_output {
|
||||
my ($instance_id) = @_;
|
||||
my $output = $ec2->get_console_output(InstanceId => $instance_id);
|
||||
return "(Last updated: ".$output->timestamp.")\n".$output->output;
|
||||
}
|
||||
|
||||
sub start_ec2_instance {
|
||||
# return [ {
|
||||
# reservation_id => 'r-12345678',
|
||||
# instance_id => 'i-12345678',
|
||||
# instance_state => 'pending',
|
||||
# image_id => 'ami-12345678',
|
||||
# dns_name => '',
|
||||
# launch_time => '2008-12-30T17:14:22.000Z',
|
||||
# reason => '',
|
||||
# } ];
|
||||
|
||||
my $user_data = create_user_data();
|
||||
|
||||
my $reservation = $ec2->run_instances(
|
||||
ImageId => $config{image_id},
|
||||
MinCount => 1,
|
||||
MaxCount => 1,
|
||||
KeyName => $config{key_name},
|
||||
SecurityGroup => $config{security_group},
|
||||
UserData => encode_base64($user_data),
|
||||
InstanceType => $config{instance_type},
|
||||
'Placement.AvailabilityZone' => $config{availability_zone},
|
||||
);
|
||||
|
||||
if (ref $reservation eq 'Net::Amazon::EC2::Errors') {
|
||||
die "run_instances failed:\n".(Dumper $reservation);
|
||||
}
|
||||
|
||||
return [ map flatten_instance($reservation, $_), @{$reservation->instances_set} ];
|
||||
}
|
||||
|
||||
sub create_user_data {
|
||||
my @files = qw(run.pl run.conf);
|
||||
|
||||
my $zip = new Archive::Zip;
|
||||
for (@files) {
|
||||
$zip->addFile("$root/$_", "$_") or die "Failed to add $root/$_ to zip";
|
||||
}
|
||||
my $fh = new IO::String;
|
||||
if ($zip->writeToFileHandle($fh) != Archive::Zip::AZ_OK) {
|
||||
die "writeToFileHandle failed";
|
||||
}
|
||||
return ${$fh->string_ref};
|
||||
}
|
||||
|
||||
sub stop_ec2_instance {
|
||||
my ($instance_id) = @_;
|
||||
|
||||
# return;
|
||||
|
||||
$ec2->terminate_instances(
|
||||
InstanceId => $instance_id,
|
||||
);
|
||||
}
|
||||
|
||||
sub load_conf {
|
||||
my ($filename) = @_;
|
||||
open my $f, '<', $filename or die "Failed to open $filename: $!";
|
||||
my %c;
|
||||
while (<$f>) {
|
||||
if (/^(.+?): (.+)/) {
|
||||
$c{$1} = $2;
|
||||
}
|
||||
}
|
||||
return %c;
|
||||
}
|
8
source/tools/autobuild2/manage.conf.example
Normal file
8
source/tools/autobuild2/manage.conf.example
Normal file
@ -0,0 +1,8 @@
|
||||
aws_access_key_id: XXXXXXXXXXXXXXXXXXXX
|
||||
aws_secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||
database: manage.db
|
||||
image_id: ami-XXXXXXXX
|
||||
instance_type: m1.small
|
||||
availability_zone: us-east-1c
|
||||
key_name: XXX
|
||||
security_group: XXX
|
11
source/tools/autobuild2/manage.css
Normal file
11
source/tools/autobuild2/manage.css
Normal file
@ -0,0 +1,11 @@
|
||||
body {
|
||||
font-family: sans-serif;
|
||||
}
|
||||
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
}
|
||||
|
||||
table th, table td {
|
||||
border: 1px solid #aaa;
|
||||
}
|
21
source/tools/autobuild2/manage.sql
Normal file
21
source/tools/autobuild2/manage.sql
Normal file
@ -0,0 +1,21 @@
|
||||
CREATE TABLE activity (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
user TEXT NOT NULL,
|
||||
ip TEXT NOT NULL,
|
||||
ua TEXT NOT NULL,
|
||||
action TEXT NOT NULL, -- like "index", "start", "stop", "console"
|
||||
params TEXT -- action-dependent - typically just the instance ID
|
||||
);
|
||||
|
||||
CREATE TABLE instances (
|
||||
instance_id TEXT NOT NULL PRIMARY KEY,
|
||||
local_launch_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE state (
|
||||
key TEXT NOT NULL PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO state VALUES ('process_mutex', '');
|
||||
INSERT INTO state VALUES ('last_start', '0');
|
7
source/tools/autobuild2/run.conf.example
Normal file
7
source/tools/autobuild2/run.conf.example
Normal file
@ -0,0 +1,7 @@
|
||||
ec2_metadata_root: http://169.254.169.254/2008-09-01/meta-data/
|
||||
ebs_volume_id: vol-XXXXXXXX
|
||||
ebs_device: xvdg
|
||||
ebs_drive_letter: e
|
||||
sync: c:\bin\sync.exe
|
||||
svn_username: autobuild
|
||||
svn_password: XXXXXXXXX
|
@ -1,6 +1,6 @@
|
||||
=pod
|
||||
|
||||
This script is executed on startup (via a service) on the build server.
|
||||
This script does the EC2-specific stuff
|
||||
|
||||
It is responsible for:
|
||||
* attaching the necessary disks,
|
||||
@ -9,8 +9,7 @@ It is responsible for:
|
||||
* cleaning up at the end,
|
||||
* and saving the logs of everything that's going on.
|
||||
|
||||
This script does as little as possible, since it is necessarily frozen
|
||||
into the static machine image and is hard to update.
|
||||
i.e. everything except actually building.
|
||||
|
||||
=cut
|
||||
|
||||
@ -22,7 +21,7 @@ use Amazon::S3;
|
||||
use LWP::Simple();
|
||||
use DateTime;
|
||||
|
||||
my %config = load_conf("c:\\0ad\\autobuild\\aws.conf");
|
||||
my %config = (load_conf("c:\\0ad\\autobuild\\aws.conf"), load_conf("d:\\0ad\\autobuild\\run.conf"));
|
||||
my $timestamp = DateTime->now->iso8601;
|
||||
|
||||
my $s3 = new Amazon::S3( {
|
||||
@ -57,10 +56,7 @@ $SIG{__DIE__} = sub {
|
||||
die @_;
|
||||
};
|
||||
|
||||
$ec2 = new Net::Amazon::EC2(
|
||||
AWSAccessKeyId => $config{aws_access_key_id},
|
||||
SecretAccessKey => $config{aws_secret_access_key},
|
||||
);
|
||||
connect_to_ec2();
|
||||
|
||||
my $instance_id = get_instance_id();
|
||||
write_log("Running on instance $instance_id");
|
||||
@ -74,6 +70,8 @@ run_build_script();
|
||||
|
||||
save_buildlogs();
|
||||
|
||||
connect_to_ec2(); # in case it timed out while building
|
||||
|
||||
detach_disk();
|
||||
|
||||
write_log("Finished");
|
||||
@ -104,6 +102,15 @@ sub save_buildlogs {
|
||||
}
|
||||
}
|
||||
|
||||
sub connect_to_ec2 {
|
||||
# This might need to be called more than once, if you wait
|
||||
# so long that the original connection times out
|
||||
$ec2 = new Net::Amazon::EC2(
|
||||
AWSAccessKeyId => $config{aws_access_key_id},
|
||||
SecretAccessKey => $config{aws_secret_access_key},
|
||||
);
|
||||
}
|
||||
|
||||
sub attach_disk {
|
||||
write_log("Attaching volume $config{ebs_volume_id} as $config{ebs_device}");
|
||||
my $status = $ec2->attach_volume(
|
||||
@ -167,11 +174,18 @@ sub terminate_instance {
|
||||
sleep 60*5;
|
||||
write_log("Really terminating now");
|
||||
flush_log();
|
||||
|
||||
connect_to_ec2(); # in case it timed out while sleeping
|
||||
|
||||
while (1) {
|
||||
my $statuses = $ec2->terminate_instances(
|
||||
InstanceId => $instance_id,
|
||||
);
|
||||
write_log("Terminated");
|
||||
use Data::Dumper;
|
||||
write_log("Termination status $statuses -- ".(Dumper $statuses));
|
||||
flush_log();
|
||||
sleep 15;
|
||||
}
|
||||
}
|
||||
|
||||
sub get_instance_id {
|
105
source/tools/autobuild2/terminator.pl
Executable file
105
source/tools/autobuild2/terminator.pl
Executable file
@ -0,0 +1,105 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
=pod
|
||||
|
||||
To prevent runaway server instances eating up lots of money, this script
|
||||
is run frequently by cron and will kill any that have been running for too long.
|
||||
|
||||
(The instances attempt to terminate themselves once they're finished building,
|
||||
so typically this script won't be required.)
|
||||
|
||||
To cope with dodgy clock synchronisation, two launch times are used per instance:
|
||||
the launch_time reported by EC2's describe_instances, and the local_launch_time
|
||||
stored in SQLite by manage.cgi. The process is killed if either time exceeds
|
||||
the cutoff limit.
|
||||
|
||||
=cut
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use DBI;
|
||||
use Net::Amazon::EC2;
|
||||
use DateTime::Format::ISO8601;
|
||||
|
||||
my $root = '/var/svn/autobuild';
|
||||
|
||||
my %config = load_conf("$root/manage.conf");
|
||||
|
||||
my $dbh = DBI->connect("dbi:SQLite:dbname=$root/$config{database}", '', '', { RaiseError => 1 });
|
||||
|
||||
my $now = DateTime->now;
|
||||
|
||||
print "\n", $now->iso8601, "\n";
|
||||
|
||||
my $cutoff = DateTime::Duration->new(minutes => 55);
|
||||
|
||||
my $ec2 = new Net::Amazon::EC2(
|
||||
AWSAccessKeyId => $config{aws_access_key_id},
|
||||
SecretAccessKey => $config{aws_secret_access_key},
|
||||
);
|
||||
|
||||
my @instances;
|
||||
|
||||
my $reservations = $ec2->describe_instances();
|
||||
for my $reservation (@$reservations) {
|
||||
for my $instance (@{$reservation->instances_set}) {
|
||||
my ($local_launch_time) = $dbh->selectrow_array('SELECT local_launch_time FROM instances WHERE instance_id = ?', undef, $instance->instance_id);
|
||||
push @instances, {
|
||||
id => $instance->instance_id,
|
||||
state => $instance->instance_state->name,
|
||||
launch_time => $instance->launch_time,
|
||||
local_launch_time => $local_launch_time,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
use Data::Dumper; print Dumper \@instances;
|
||||
# @instances = ( {
|
||||
# id => 'i-12345678',
|
||||
# state => 'pending',
|
||||
# launch_time => '2008-12-30T17:14:22.000Z',
|
||||
# local_launch_time => '2008-12-30T17:14:22.000Z',
|
||||
# } );
|
||||
|
||||
for my $instance (@instances) {
|
||||
next if $instance->{state} eq 'terminated';
|
||||
|
||||
my $too_old = 0;
|
||||
my $age = $now - DateTime::Format::ISO8601->parse_datetime($instance->{launch_time});
|
||||
$too_old = 1 if DateTime::Duration->compare($age, $cutoff) > 0;
|
||||
if (defined $instance->{local_launch_time}) {
|
||||
my $local_age = $now - DateTime::Format::ISO8601->parse_datetime($instance->{local_launch_time});
|
||||
$too_old = 1 if DateTime::Duration->compare($local_age, $cutoff) > 0;
|
||||
}
|
||||
next unless $too_old;
|
||||
|
||||
print "Terminating $instance->{id}, launched at $instance->{launch_time} / ", ($instance->{local_launch_time} || ''), "\n";
|
||||
|
||||
log_action('terminate', $instance->{id});
|
||||
|
||||
$ec2->terminate_instances(
|
||||
InstanceId => $instance->{id},
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
$dbh->disconnect;
|
||||
|
||||
sub log_action {
|
||||
my ($action, $params) = @_;
|
||||
$dbh->do('INSERT INTO activity (user, ip, ua, action, params) VALUES (?, ?, ?, ?, ?)',
|
||||
undef, 'local', '', '', $action, $params);
|
||||
}
|
||||
|
||||
sub load_conf {
|
||||
my ($filename) = @_;
|
||||
open my $f, '<', $filename or die "Failed to open $filename: $!";
|
||||
my %c;
|
||||
while (<$f>) {
|
||||
if (/^(.+?): (.+)/) {
|
||||
$c{$1} = $2;
|
||||
}
|
||||
}
|
||||
return %c;
|
||||
}
|
Loading…
Reference in New Issue
Block a user