# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html

use strict;

use warnings;

use Getopt::Long;

use Data::Dumper;

use Time::Local;

use File::Basename;

use File::Path;

use File::Find;

use LWP::UserAgent;

use HTTP::Request;

use Encode;

use Encode::CN;

use JSON;

use URI::Escape;

use LWP::Simple;

use Term::ANSIColor qw(:constants);

$Term::ANSIColor::AUTORESET = 1;

#---Global attributes---

my $rst = 0;

my $retries = 5; # Try this many times to get response

my $check_result_str="``CI CHECK RESULT`` : ";

my $last_func_start = timelocal(localtime());

my $GITHUB_API = "https://api.github.com";

#--------------------------------------------------------

# Fuction name: runcmd

# Description: run a command after 'cmd' label in one case

# Attributes:

# Return code:

# $::RUNCMD_RC : the return code of command

# @$outref : the output of command

#--------------------------------------------------------

sub runcmd

{

my ($cmd) = @_;

my $rc = 0;

$::RUNCMD_RC = 0;

my $outref = [];

@$outref = `$cmd 2>&1`;

if ($?)

{

$rc = $?;

$rc = $rc >> 8;

$::RUNCMD_RC = $rc;

}

chomp(@$outref);

return @$outref;

}

#--------------------------------------------------------

# Fuction name: get_files_recursive

# Description: Search all file in one directory recursively

# Attributes:

# $dir (input attribute)

# The target scan directory

# $files_path_ref (output attribute)

# the reference of array where save all vaild files under $dir

# Return code:

#--------------------------------------------------------

sub get_files_recursive

{

my $dir = shift;

my $files_path_ref = shift;

my $fd = undef;

if(!opendir($fd, $dir)){

print "[get_files_recursive]: failed to open $dir :$!\n";

return 1;

}

for (; ;)

{

my $direntry = readdir($fd);

last unless (defined($direntry));

next if ($direntry =~ m/^\.\w*/);

next if ($direntry eq '..');

my $target = "$dir/$direntry";

if (-d $target) {

get_files_recursive($target, $files_path_ref);

} else {

push(@{$files_path_ref}, glob("$target\n"));

}

}

closedir($fd);

return 0;

}

#--------------------------------------------------------

# Fuction name: check_pr_format

# Description:

# Attributes:

# Return code:

#--------------------------------------------------------

sub check_pr_format{

if($ENV{'TRAVIS_EVENT_TYPE'} eq "pull_request"){

my $pr_url = "$GITHUB_API/repos/$ENV{'TRAVIS_REPO_SLUG'}/pulls/$ENV{'TRAVIS_PULL_REQUEST'}";

my $pr_url_resp;

my $counter = 1;

while($counter <= $retries) {

$pr_url_resp = get($pr_url);

if ($pr_url_resp) {

last; # Got response, no more retries

} else {

sleep($counter*2); # Sleep and try again

print "[check_pr_format] $counter Did not get response, sleeping ". $counter*2 . "\n";

$counter++;

}

}

unless ($pr_url_resp) {

print "[check_pr_format] After $retries retries, not able to get response from $pr_url \n";

# Failed after trying a few times, return error

return $counter;

}

my $pr_content = decode_json($pr_url_resp);

my $pr_title = $pr_content->{title};

my $pr_body = $pr_content->{body};

my $pr_milestone = $pr_content->{milestone};

my $pr_labels_len = @{$pr_content->{labels}};

#print "[check_pr_format] Dumper pr_content:\n";

#print Dumper $pr_content;

print "[check_pr_format] pr title = $pr_title\n";

print "[check_pr_format] pr body = $pr_body \n";

my $checkrst="";

if(! $pr_title){

$checkrst.="Missing title.";

}

if(! $pr_body){

$checkrst.="Missing description.";

}

if(! $pr_milestone){

$checkrst.="Missing milestone.";

}

if(! $pr_labels_len){

$checkrst.="Missing labels.";

}

# Guard against root user making commits

$checkrst.=check_commit_owner('root');

if(length($checkrst) == 0){

$check_result_str .= "> **PR FORMAT CORRECT**";

send_back_comment("$check_result_str");

}else{

# Warning if missing milestone or labels, others are errors

if($checkrst =~ /milestone/ || $checkrst =~ /labels/){

$check_result_str .= "> **PR FORMAT WARNING** : $checkrst";

send_back_comment("$check_result_str");

}else{

$check_result_str .= "> **PR FORMAT ERROR** : $checkrst";

send_back_comment("$check_result_str");

return 1;

}

}

}

return 0;

}

#--------------------------------------------------------

# Fuction name: check_commit_owner

# Description: Verify commits are not done by specified user

# Attributes: user login to reject

# Return:

# Error string -User rejected,

# Empty string -User not rejected

#--------------------------------------------------------

sub check_commit_owner{

my $invalid_user = shift;

if($ENV{'TRAVIS_EVENT_TYPE'} eq "pull_request"){

my $commits_content;

my $commits_len = 0;

my $json = new JSON;

my $commits_url = "$GITHUB_API/repos/$ENV{'TRAVIS_REPO_SLUG'}/pulls/$ENV{'TRAVIS_PULL_REQUEST'}/commits";

my $commits_url_resp;

my $counter = 1;

while($counter <= $retries) {

$commits_url_resp = get($commits_url);

if ($commits_url_resp) {

last; # Got response, no more retries

} else {

sleep($counter*2); # Sleep and try again

print "[check_commit_owner] $counter Did not get response, sleeping ". $counter*2 . "\n";

$counter++;

}

}

if ($commits_url_resp) {

$commits_content = $json->decode($commits_url_resp);

$commits_len = @$commits_content;

} else {

print "[check_commit_owner] After $retries retries, not able to get response from $commits_url \n";

return "Unable to verify login of committer.";

}

if($commits_len > 0) {

foreach my $commit (@{$commits_content}){

my $committer = $commit->{committer};

my $committer_login = $committer->{login};

print "[check_commit_owner] Committer login $committer_login \n";

if($committer_login =~ /^$invalid_user$/) {

# Committer logins matches

return "Commits by $invalid_user not allowed";

}

}

}

}

return "";

}

#--------------------------------------------------------

# Fuction name: send_back_comment

# Description: Append to comment of the PR passed $message

# Attributes: Message to append to PR

# Return code:

#--------------------------------------------------------

sub send_back_comment{

my $message = shift;

my $comment_url = "$GITHUB_API/repos/$ENV{'TRAVIS_REPO_SLUG'}/issues/$ENV{'TRAVIS_PULL_REQUEST'}/comments";

my $json = new JSON;

my $comment_len = 0;

my $comment_content;

my $comment_url_resp;

my $counter = 1;

while($counter <= $retries) {

$comment_url_resp = get($comment_url);

if ($comment_url_resp) {

last; # Got response, no more retries

} else {

sleep($counter*2); # Sleep and try again

print "[send_back_comment] $counter Did not get response, sleeping ". $counter*2 . "\n";

$counter++;

}

}

unless ($comment_url_resp) {

print "[send_back_comment] After $retries retries, not able to get response from $comment_url \n";

# Failed after trying a few times, return

return;

}

print "\n\n>>>>>Dumper comment_url_resp:\n";

print Dumper $comment_url_resp;

$comment_content = $json->decode($comment_url_resp);

$comment_len = @$comment_content;

my $post_url = $comment_url;

my $post_method = "POST";

if($comment_len > 0){

foreach my $comment (@{$comment_content}){

if($comment->{'body'} =~ /CI CHECK RESULT/) {

$post_url = $comment->{'url'};

$post_method = "PATCH";

}

}

}

print "[send_back_comment] method = $post_method to $post_url. Message = $message\n";

if ( $ENV{'xcatbotuser'} and $ENV{'xcatbotpw'}) {

`curl -u "$ENV{'xcatbotuser'}:$ENV{'xcatbotpw'}" -X $post_method -d '{"body":"$message"}' $post_url`;

}

else {

print "Not able to update pull request with message: $message\n";

}

}

#--------------------------------------------------------

# Fuction name: build_xcat_core

# Description:

# Attributes:

# Return code:

#--------------------------------------------------------

sub build_xcat_core{

my @output;

#my @cmds = ("gpg --list-keys",

# "sed -i '/SignWith: /d' $ENV{'PWD'}/build-ubunturepo");

#foreach my $cmd (@cmds){

# print "[build_xcat_core] running $cmd\n";

# @output = runcmd("$cmd");

# if($::RUNCMD_RC){

# print "[build_xcat_core] $cmd ....[Failed]\n";

# send_back_comment("> **BUILD ERROR** : $cmd failed. Please click ``Details`` label in ``Merge pull request`` box for detailed information");

# return 1;

# }

#}

my $cmd = "sudo ./build-ubunturepo -c UP=0 BUILDALL=1 GPGSIGN=0";

@output = runcmd("$cmd");

print ">>>>>Dumper the output of '$cmd'\n";

print Dumper \@output;

if($::RUNCMD_RC){

my $lastline = $output[-1];

$lastline =~ s/[\r\n\t\\"']*//g;

print "[build_xcat_core] $cmd ....[Failed]\n";

#print ">>>>>Dumper the output of '$cmd'\n";

#print Dumper \@output;

$check_result_str .= "> **BUILD ERROR**, Please click ``Details`` label in ``Merge pull request`` box for detailed information";

send_back_comment("$check_result_str");

return 1;

}else{

print "[build_xcat_core] $cmd ....[Pass]\n";

$check_result_str .= "> **BUILD SUCCESSFUL** ";

send_back_comment("$check_result_str");

}

# my $buildpath ="/home/travis/build/xcat-core/";

# my @buildfils = ();

# get_files_recursive("$buildpath", \@buildfils);

# print "\n-----------Dumper build files-----------\n";

# print Dumper \@buildfils;

return 0;

}

#--------------------------------------------------------

# Fuction name: install_xcat

# Description:

# Attributes:

# Return code:

#--------------------------------------------------------

sub install_xcat{

my @cmds = ("cd ./../../xcat-core && sudo ./mklocalrepo.sh",

"sudo chmod 777 /etc/apt/sources.list",

"sudo echo \"deb [arch=amd64 allow-insecure=yes] http://xcat.org/files/xcat/repos/apt/devel/xcat-dep xenial main\" >> /etc/apt/sources.list",

"sudo echo \"deb [arch=ppc64el allow-insecure=yes] http://xcat.org/files/xcat/repos/apt/devel/xcat-dep xenial main\" >> /etc/apt/sources.list",

"sudo wget -q -O - \"http://xcat.org/files/xcat/repos/apt/apt.key\" | sudo apt-key add -",

"sudo apt-get -qq update");

my @output;

foreach my $cmd (@cmds){

print "[install_xcat] running $cmd\n";

@output = runcmd("$cmd");

if($::RUNCMD_RC){

print RED "[install_xcat] $cmd. ...[Failed]\n";

print "[install_xcat] error message:\n";

print Dumper \@output;

$check_result_str .= "> **INSTALL XCAT ERROR** : Please click ``Details`` label in ``Merge pull request`` box for detailed information ";

send_back_comment("$check_result_str");

return 1;

}

}

my $cmd = "sudo apt-get install xcat --allow-remove-essential --allow-unauthenticated";

@output = runcmd("$cmd");

#print ">>>>>Dumper the output of '$cmd'\n";

#print Dumper \@output;

if($::RUNCMD_RC){

my $lastline = $output[-1];

$lastline =~ s/[\r\n\t\\"']*//g;

print "[install_xcat] $cmd ....[Failed]\n";

print ">>>>>Dumper the output of '$cmd'\n";

print Dumper \@output;

$check_result_str .= "> **INSTALL XCAT ERROR** : Please click ``Details`` label in ``Merge pull request`` box for detailed information";

send_back_comment("$check_result_str");

return 1;

}else{

print "[install_xcat] $cmd ....[Pass]\n";

print "\n------Config xcat and verify xcat is working correctly-----\n";

@cmds = ("sudo -s /opt/xcat/share/xcat/scripts/setup-local-client.sh -f travis",

"sudo -s /opt/xcat/sbin/chtab priority=1.1 policy.name=travis policy.rule=allow",

". /etc/profile.d/xcat.sh && tabdump policy",

". /etc/profile.d/xcat.sh && tabdump site",

". /etc/profile.d/xcat.sh && lsxcatd -a",

"ls /opt/xcat/sbin",

"service xcatd status");

my $ret = 0;

foreach my $cmd (@cmds){

print "\n[install_xcat] running $cmd.....\n";

@output = runcmd("$cmd");

print Dumper \@output;

if($::RUNCMD_RC){

print RED "[install_xcat] $cmd. ...[Failed]\n";

#print Dumper \@output;

$ret = 1;

}else{

print "[install_xcat] $cmd....[Pass]\n";

}

}

if($ret){

$check_result_str .= "> **INSTALL XCAT ERROR** : Please click ``Details`` label in ``Merge pull request`` box for detailed information";

send_back_comment("$check_result_str");

return 1;

}

$check_result_str .= "> **INSTALL XCAT SUCCESSFUL**";

send_back_comment("$check_result_str");

}

return 0;

}

#--------------------------------------------------------

# Fuction name: check_syntax

# Description:

# Attributes:

# Return code:

#--------------------------------------------------------

sub check_syntax{

my @output;

my @syntax_err;

my $ret = 0;

my @target_dirs=("/opt/xcat",

"/install");

foreach my $dir (@target_dirs){

my @files = ();

get_files_recursive("$dir", \@files);

foreach my $file (@files) {

next if($file =~ /\/opt\/xcat\/share\/xcat\/netboot\/genesis\//);

next if($file =~ /\/opt\/xcat\/probe\//);

@output = runcmd("file $file");

if($output[0] =~ /perl /i){

@output = runcmd("sudo bash -c '. /etc/profile.d/xcat.sh && perl -I /opt/xcat/lib/perl -I /opt/xcat/lib -I /usr/lib/perl5 -I /usr/share/perl -c $file'");

if($::RUNCMD_RC){

push @syntax_err, @output;

$ret = 1;

}

#}elsif($output[0] =~ /shell/i){

# @output = runcmd("sudo bash -c '. /etc/profile.d/xcat.sh && sh -n $file'");

# if($::RUNCMD_RC){

# push @syntax_err, @output;

# $ret = 1;

# }

}

}

}

if(@syntax_err){

print "[check_syntax] syntax checking ....[Failed]\n";

print "[check_syntax] Dumper error message:\n";

print Dumper @syntax_err;

$check_result_str .= "> **CODE SYNTAX ERROR** : Please click ``Details`` label in ``Merge pull request`` box for detailed information";

send_back_comment("$check_result_str");

}else{

print "[check_syntax] syntax checking ....[Pass]\n";

$check_result_str .= "> **CODE SYNTAX CORRECT**";

send_back_comment("$check_result_str");

}

return $ret;

}

#--------------------------------------------------------

# Fuction name: run_fast_regression_test

# Description:

# Attributes:

# Return code:

#--------------------------------------------------------

sub run_fast_regression_test{

my $cmd = "sudo apt-get install xcat-test --allow-remove-essential --allow-unauthenticated";

my @output = runcmd("$cmd");

if($::RUNCMD_RC){

print RED "[run_fast_regression_test] $cmd ....[Failed]\n";

print Dumper \@output;

return 1;

}else{

print "[run_fast_regression_test] $cmd .....:\n";

print Dumper \@output;

}

$cmd = "sudo bash -c '. /etc/profile.d/xcat.sh && xcattest -h'";

@output = runcmd("$cmd");

if($::RUNCMD_RC){

print RED "[run_fast_regression_test] $cmd ....[Failed]\n";

print "[run_fast_regression_test] error dumper:\n";

print Dumper \@output;

return 1;

}else{

print "[run_fast_regression_test] $cmd .....:\n";

print Dumper \@output;

}

my $hostname = `hostname`;

chomp($hostname);

print "hostname = $hostname\n";

my $conf_file = "$ENV{'PWD'}/regression.conf";

$cmd = "echo '[System]' > $conf_file; echo 'MN=$hostname' >> $conf_file; echo '[Table_site]' >> $conf_file; echo 'key=domain' >>$conf_file; echo 'value=pok.stglabs.ibm.com' >> $conf_file";

@output = runcmd("$cmd");

if($::RUNCMD_RC){

print RED "[run_fast_regression_test] $cmd ....[Failed]";

print "[run_fast_regression_test] error dumper:\n";

print Dumper \@output;

return 1;

}

print "Dumper regression conf file:\n";

@output = runcmd("cat $conf_file");

print Dumper \@output;

$cmd = "sudo bash -c '. /etc/profile.d/xcat.sh && xcattest -s \"ci_test\" -l'";

my @caseslist = runcmd("$cmd");

if($::RUNCMD_RC){

print RED "[run_fast_regression_test] $cmd ....[Failed]\n";

print "[run_fast_regression_test] error dumper:\n";

print Dumper \@caseslist;

return 1;

}else{

print "[run_fast_regression_test] $cmd .....:\n";

print Dumper \@caseslist;

}

#my @caseslist = runcmd("sudo bash -c '. /etc/profile.d/xcat.sh && xcattest -l caselist -b MN_basic.bundle'");

my $casenum = @caseslist;

my $x = 0;

my @failcase;

my $passnum = 0;

my $failnum = 0;

foreach my $case (@caseslist){

++$x;

$cmd = "sudo bash -c '. /etc/profile.d/xcat.sh && xcattest -f $conf_file -t $case'";

print "[run_fast_regression_test] run $x: $cmd\n";

@output = runcmd("$cmd");

#print Dumper \@output;

for(my $i = $#output; $i>-1; --$i){

if($output[$i] =~ /------END::(.+)::Failed/){

push @failcase, $1;

++$failnum;

print Dumper \@output;

last;

}elsif ($output[$i] =~ /------END::(.+)::Passed/){

++$passnum;

last;

}

}

}

if($failnum){

my $log_str = join (",", @failcase );

$check_result_str .= "> **FAST REGRESSION TEST Failed**: Totalcase $casenum Passed $passnum Failed $failnum FailedCases: $log_str. Please click ``Details`` label in ``Merge pull request`` box for detailed information";

send_back_comment("$check_result_str");

return 1;

}else{

$check_result_str .= "> **FAST REGRESSION TEST Successful**: Totalcase $casenum Passed $passnum Failed $failnum";

send_back_comment("$check_result_str");

}

return 0;

}

#--------------------------------------------------------

# Fuction name: mark_time

# Description:

# Attributes:

# Return code:

#--------------------------------------------------------

sub mark_time{

my $func_name=shift;

my $nowtime = timelocal(localtime());

my $nowtime_str = scalar(localtime());

my $duration = $nowtime - $last_func_start;

$last_func_start = $nowtime;

print "[mark_time] $nowtime_str, ElapsedTime of $func_name is $duration s\n";

}

#===============Main Process=============================

#Dumper Travis Environment Attribute

print GREEN "\n------ Travis Environment Attributes ------\n";

my @travis_env_attr = ("TRAVIS_REPO_SLUG",

"TRAVIS_BRANCH",

"TRAVIS_EVENT_TYPE",

"TRAVIS_PULL_REQUEST",

"GITHUB_TOKEN",

"USERNAME",

"PASSWORD",

"PWD");

foreach (@travis_env_attr){

if($ENV{$_}) {

print "$_ = '$ENV{$_}'\n";

} else {

print "$_ = ''\n";

}

}

my @os_info = runcmd("cat /etc/os-release");

print "Current OS information:\n";

print Dumper \@os_info;

my @perl_vserion = runcmd("perl -v");

print "Current perl information:\n";

print Dumper \@perl_vserion;

#my @sh_version = runcmd("sudo bash -c 'sh --version'");

#print "Current sh information:\n";

#print Dumper \@sh_version;

my @disk = runcmd("df -h");

print "Disk information:\n";

print Dumper \@disk;

# Hacking the netmask. Not sure if we need to recover it after finish xcattest

# Note: Here has an assumption from Travis VM: only 1 UP Ethernet interface available (CHANGEME if it not as is)

my @intfinfo = runcmd("ip -o link |grep 'link/ether'|grep 'state UP' |awk -F ':' '{print \$2}'|head -1");

foreach my $nic (@intfinfo) {

print "Hacking the netmask length to 16 if it is 32: $nic\n";

runcmd("ip -4 addr show $nic|grep 'inet'|grep -q '/32' && sudo ip addr add \$(hostname -I|awk '{print \$1}')/16 dev $nic");

}

my @ipinfo = runcmd("ip addr");

print "Networking information:\n";

print Dumper \@ipinfo;

#Start to check the format of pull request

$last_func_start = timelocal(localtime());

print GREEN "\n------ Checking Pull Request Format ------\n";

$rst = check_pr_format();

my $redo_check_pr = 0;

if($rst){

if($rst <= $retries) {

print RED "Check of pull request format failed\n";

exit $rst;

}

$redo_check_pr = 1;

}

mark_time("check_pr_format");

#Start to build xcat core

print GREEN "\n------ Building xCAT core package ------\n";

$rst = build_xcat_core();

if($rst){

print RED "Build of xCAT core package failed\n";

exit $rst;

}

mark_time("build_xcat_core");

#Start to install xcat

print GREEN "\n------Installing xCAT ------\n";

$rst = install_xcat();

if($rst){

print RED "Install of xCAT failed\n";

exit $rst;

}

mark_time("install_xcat");

#Check the syntax of changing code

print GREEN "\n------ Checking the syntax of changed code------\n";

$rst = check_syntax();

if($rst){

print RED "Check syntax of changed code failed\n";

exit $rst;

}

mark_time("check_syntax");

#run fast regression

print GREEN "\n------Running fast regression test ------\n";

$rst = run_fast_regression_test();

if($rst){

print RED "Run of fast regression test failed\n";

exit $rst;

}

mark_time("run_fast_regression_test");

if ($redo_check_pr) {

print GREEN "\n------ Checking Pull Request Format ------\n";

$rst = check_pr_format();

if($rst){

print RED "Check of pull request format failed\n";

exit $rst;

}

mark_time("check_pr_format");

}

exit 0;