VCMI automated testing PoC

test.go

package main

import (
	"fmt"
	_"flag"
	_"time"
	"strings"
	"compress/gzip"
	"io/ioutil"
	"path/filepath"
	"sync"
	"bufio"
	"os"
	"os/exec"
	"log"
	"encoding/binary"
	"strconv"
)

const firstPort = 55500
const maxConcurrency = 3

func main() {
	jobs := make(chan string)
	
    //Spawning 4 goroutines
    var wg sync.WaitGroup
	for i := 0; i < maxConcurrency; i++ {
		wg.Add(1)
		go testWorker(firstPort+i, &wg, jobs)
    }
    
	files, _ := filepath.Glob("/home/i/.local/share/vcmi/Maps/*.h3m")
	for _, file := range files {
        jobs <- file
	}
    close(jobs)
	
    wg.Wait()
}

func testWorker(port int, wg *sync.WaitGroup, jobs <-chan string) {
	defer wg.Done()
    for testname := range jobs {
		// whatever processing
		_, _, err, expectWin := parsemap(testname)
		if err {
			fmt.Println("WRONG TEST: ", testname)
		} else {
			mapname := "Maps/" + filepath.Base(testname); 
//			out, _ := exec.Command("timeout", "10", "vcmiclient", "--headless", "--testmap", mapname, "--testingport", strconv.Itoa(port), "--testingfileprefix", "test_").Output()
			out, _ := exec.Command("vcmiclient", "--headless", "--testmap", mapname, "--testingport", strconv.Itoa(port), "--testingfileprefix", "test_", "1>&2").Output()
//			fmt.Println(string(out));
			crash := strings.Index(string(out), "Backtrace:")
			win := strings.Index(string(out), "I heard that player 0 (red) won")
			lose := strings.Index(string(out), "I heard that player 0 (red) lost")
			if crash > -1 {
				crashlog := "/tmp/" + strconv.Itoa(port) + "_" + filepath.Base(testname) + ".log"
				ioutil.WriteFile(crashlog, out, 0644)
				fmt.Println(mapname, ": CRASHED \n LOG: ", crashlog)
			} else if (expectWin && win > -1) || (!expectWin && lose > -1) {
				fmt.Println(mapname, ": PASS")
			} else {
				fmt.Println(mapname, ": FAIL")
			}
		}
	}
}

func parsemap(file string) (title string, description string, somethingwrong bool, expectedResult bool) {
	f, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	gr, err := gzip.NewReader(f)
	if err != nil {
		log.Fatal(err)
	}
	reader := bufio.NewReader(gr)

	// Skip 10 bytes: 4 version, 1 anyplayers, 4 size, 1 twolevel
	reader.Discard(10)
	sizeUI32 := make([]byte, 4)

	// Read map title
	reader.Read(sizeUI32)
	sizeTitle := int(binary.LittleEndian.Uint32(sizeUI32))
	dataTitle := make([]byte, sizeTitle)
	reader.Read(dataTitle)
	title = string(dataTitle)

	// Read map description
	reader.Read(sizeUI32)
	sizeDescription := int(binary.LittleEndian.Uint32(sizeUI32))
	dataDescription := make([]byte, sizeDescription) 
	reader.Read(dataDescription)
	description = string(dataDescription)
	
	defer gr.Close()
	
	win := strings.Index(description, "MUST_WIN")
	lose := strings.Index(description, "MUST_LOSE")
	if win > -1 {
		expectedResult = true
	} else if lose > -1 {
		expectedResult = false
	} else {
		somethingwrong = true
	}
	
	return title, description, somethingwrong, expectedResult
}

test.php

#!/usr/bin/php
<?php
$flags = array(
'MUST_WIN', 'MUST_LOSE', 'DEADLINE_WEEK', 'DEADLINE_4WEEKS', 'DEADLINE_CUSTOM', 'REPEAT_CAN_WIN', 'REPEAT_CAN_LOSE', 'REPEAT_NEVER_LOSE', 'REPEAT_NEVER_WIN', 'H3FAIL', 'H3CRASH', 'H3BUG', 'DUMMY_TEST', 'BATTLE_IMPORTANT', 'LONG_TEST', 'TEAM_TEST', 'AIBEHAVIOR_IMPORTANT', 'CHEAT_TEST', 'FOW_TEST'
);


$files = explode(PHP_EOL, shell_exec("find ./Maps -type f | sort"));
foreach($files as $id => $file)
{
	if($id <= 0)
		continue;
	if(strlen($file) < 3)
		continue;
//	echo "Map ". $file .":".PHP_EOL;
	echo "Map ". $file .": ";
	$opts = parsemap($file);//Wasteland.h3m");
	$filename = str_replace(array('./', '.h3m'), '', $file);
	$cmd = "timeout 60 vcmiclient --noGUI --testingport=44433 --testingfileprefix=autotest_ --testmap=". $filename. " 2>&1";
	echo $cmd;
	$result = shell_exec($cmd);
//	echo "Expected: ". ($opts['expectwin']?"WIN":"LOSS").PHP_EOL.
//			"Result: ". ((strpos($result, 'I heard that player 0 (red) won'))?"WIN":"LOSS").PHP_EOL;
	echo $result;
//	exit;
	if((iswon($result) && $opts['expectwin']) || (islost($result) && !$opts['expectwin']))		
		echo "PASS";
	else if(!iswon($result) && !islost($result))
		echo "TIMEOUT";
	else 
		echo "FAIL";
	
	echo PHP_EOL;
}

function iswon($result)
{
	return FALSE !== strpos($result, 'I heard that player 0 (red) won');
}

function islost($result)
{
	return FALSE !== strpos($result, 'I heard that player 0 (red) lost');
}

function parsemap($mapname)
{
	$opts = array();
	global $flags;
	$file = gzdecode(file_get_contents($mapname));
	
	foreach($flags as $flag)
	{
		if(FALSE === strpos($file, $flag))
			continue;
//		else
//			echo "Found flag: ". $flag .PHP_EOL;

		switch($flag)
		{
			case 'MUST_WIN':
				$opts['expectwin'] = 1;
				break;
			case 'MUST_LOSE':
				$opts['expectwin'] = 0;
				break;
			case 'DEADLINE_WEEK':
				break;
			case 'DEADLINE_4WEEKS':
				break;
			case 'DEADLINE_CUSTOM':
				break;
			case 'REPEAT_CAN_WIN':
				break;
			case 'REPEAT_CAN_LOSE':
				break;
			case 'REPEAT_NEVER_LOSE':
				break;
			case 'REPEAT_NEVER_WIN':
				break;
			case 'H3FAIL':
				break;
			case 'H3CRASH':
				break;
			case 'H3BUG':
				break;
			case 'DUMMY_TEST':
				break;
			case 'BATTLE_IMPORTANT':
				break;
			case 'LONG_TEST':
				break;
			case 'TEAM_TEST':
				break;
			case 'AIBEHAVIOR_IMPORTANT':
				break;
			case 'CHEAT_TEST':
				break;
			case 'FOW_TEST':
				break;
		}
	}
	
	return $opts;
}