From 60599c8d79dc7b87db85c546b89ca405d97f1303 Mon Sep 17 00:00:00 2001 From: adamdebek Date: Wed, 24 Aug 2022 12:09:11 +0200 Subject: [PATCH] lsb-vsx: enable running testsuite via trunner JIRA: CI-139 --- lsb_vsx/Makefile | 4 + lsb_vsx/lsb_vsx.py | 52 +++++ lsb_vsx/run-test.c | 68 ++++++ lsb_vsx/test.yaml | 499 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 623 insertions(+) create mode 100644 lsb_vsx/Makefile create mode 100644 lsb_vsx/lsb_vsx.py create mode 100644 lsb_vsx/run-test.c create mode 100644 lsb_vsx/test.yaml diff --git a/lsb_vsx/Makefile b/lsb_vsx/Makefile new file mode 100644 index 00000000..5ae6dd7d --- /dev/null +++ b/lsb_vsx/Makefile @@ -0,0 +1,4 @@ +NAME := run-test +LOCAL_SRCS := run-test.c + +include $(binary.mk) diff --git a/lsb_vsx/lsb_vsx.py b/lsb_vsx/lsb_vsx.py new file mode 100644 index 00000000..843281de --- /dev/null +++ b/lsb_vsx/lsb_vsx.py @@ -0,0 +1,52 @@ +from typing import Optional + +from trunner.ctx import TestContext +from trunner.dut import Dut +from trunner.types import Status, TestResult + + +def harness(dut: Dut, ctx: TestContext, result: TestResult) -> Optional[TestResult]: + tc_start = r".+?\|TP Start\r+\n" + msg_re = r".+\|(?P.*?)\r+\n" + result_re = r"(.+?)\|(?PPASS|FAIL|UNRESOLVED|UNSUPPORTED|NOTINUSE|UNTESTED)\r+\n" + final_re = r"(.+?)\|TC End.+?\r+\n" + + stats = {"FAIL": 0, "IGNORE": 0, "PASS": 0} + msg = "" + msg_lines = [] + results = [] + tc_num = 0 + get_msg = False + + dut.expect("Config End\r+\n") + + while True: + idx = dut.expect([tc_start, final_re, result_re, msg_re], timeout=1000) + parsed = dut.match.groupdict() + + if idx == 0: + get_msg = True + elif idx == 1: + break + elif idx == 2: + # Right now, not supported statuses treat as PASS + if parsed["status"] in ("UNRESOLVED", "UNSUPPORTED", "NOTINUSE", "UNTESTED"): + parsed["status"] = "PASS" + + status = Status.from_str(parsed["status"]) + tc_num += 1 + subname = f"testcase {tc_num}" + if msg_lines: + msg += msg_lines[0] + for line in msg_lines[1:]: + msg += "\n" + " " * 26 + line.lstrip() + result.add_subresult(subname, status, msg) + + stats[parsed["status"]] += 1 + results.append(parsed) + get_msg, msg_lines, msg = False, [], "" + elif idx == 3 and get_msg is True: + msg_lines.append(parsed["msg_line"]) + + status = Status.FAIL if stats["FAIL"] != 0 else Status.OK + return TestResult(status=status) diff --git a/lsb_vsx/run-test.c b/lsb_vsx/run-test.c new file mode 100644 index 00000000..b90bd5af --- /dev/null +++ b/lsb_vsx/run-test.c @@ -0,0 +1,68 @@ +/* + * Phoenix-RTOS + * + * lsb_vsx test launcher + * + * Copyright 2024 Phoenix Systems + * Author: Adam Debek + * + * This file is part of Phoenix-RTOS. + * + * %LICENSE% + */ + +#include +#include +#include +#include + + +int main(int argc, char **argv) +{ + const char *cmd = "/usr/bin/tcc -j - -e -l "; + char test_cmd[128]; + int ret; + + if (argc != 2) { + fprintf(stderr, "No path to test executable provided\n"); + exit(EXIT_FAILURE); + } + else { + strcpy(test_cmd, cmd); + strcat(test_cmd, argv[1]); + } + + if (chdir("/root/lsb_vsx/test_sets") < 0) { + perror("chdir"); + exit(EXIT_FAILURE); + } + + if (setenv("TET_ROOT", "/root/lsb_vsx", 0) < 0) { + fprintf(stderr, "setenv() - setting \"TET_ROOT\" failed\n"); + exit(EXIT_FAILURE); + } + + if (setenv("TET_EXECUTE", "/root/lsb_vsx/test_sets/TESTROOT", 0) < 0) { + fprintf(stderr, "setenv() - setting \"TET_EXECUTE\" failed\n"); + exit(EXIT_FAILURE); + } + + if ((ret = system(test_cmd)) < 0) { + perror("system"); + exit(EXIT_FAILURE); + } + + if (WIFEXITED(ret)) { + int exit_status = WEXITSTATUS(ret); + if (exit_status != 0) { + fprintf(stderr, "Error: Command exited with status %d\n", exit_status); + exit(EXIT_FAILURE); + } + } + else { + fprintf(stderr, "Error: Command did not exit normally\n"); + exit(EXIT_FAILURE); + } + + return EXIT_SUCCESS; +} diff --git a/lsb_vsx/test.yaml b/lsb_vsx/test.yaml new file mode 100644 index 00000000..7c3c1fa9 --- /dev/null +++ b/lsb_vsx/test.yaml @@ -0,0 +1,499 @@ +test: + harness: lsb_vsx.py + nightly: true + targets: + value: [ia32-generic-qemu] + + tests: + - name: T.isalnum + execute: run-test /tset/ANSI.os/charhandle/isalnum/T.isalnum + - name: T.isalpha + execute: run-test /tset/ANSI.os/charhandle/isalpha/T.isalpha + - name: T.iscntrl + execute: run-test /tset/ANSI.os/charhandle/iscntrl/T.iscntrl + - name: T.isdigit + execute: run-test /tset/ANSI.os/charhandle/isdigit/T.isdigit + - name: T.isgraph + execute: run-test /tset/ANSI.os/charhandle/isgraph/T.isgraph + - name: T.islower + execute: run-test /tset/ANSI.os/charhandle/islower/T.islower + - name: T.isprint + execute: run-test /tset/ANSI.os/charhandle/isprint/T.isprint + - name: T.ispunct + execute: run-test /tset/ANSI.os/charhandle/ispunct/T.ispunct + - name: T.isspace + execute: run-test /tset/ANSI.os/charhandle/isspace/T.isspace + - name: T.isupper + execute: run-test /tset/ANSI.os/charhandle/isupper/T.isupper + - name: T.isxdigit + execute: run-test /tset/ANSI.os/charhandle/isxdigit/T.isxdigit + - name: T.tolower + execute: run-test /tset/ANSI.os/charhandle/tolower/T.tolower + - name: T.toupper + execute: run-test /tset/ANSI.os/charhandle/toupper/T.toupper + - name: T.assert + execute: run-test /tset/ANSI.os/diagnostics/assert/T.assert + - name: T.abort + ignore: True + execute: run-test T.abort + - name: T.abs + execute: run-test /tset/ANSI.os/genuts/abs/T.abs + - name: T.atof + execute: run-test /tset/ANSI.os/genuts/atof/T.atof + - name: T.atoi + execute: run-test /tset/ANSI.os/genuts/atoi/T.atoi + - name: T.atol + execute: run-test /tset/ANSI.os/genuts/atol/T.atol + - name: T.bsearch + execute: run-test /tset/ANSI.os/genuts/bsearch/T.bsearch + - name: T.calloc + execute: run-test /tset/ANSI.os/genuts/calloc/T.calloc + - name: T.exit + ignore: True + execute: run-test T.exit + - name: T.free + execute: run-test /tset/ANSI.os/genuts/free/T.free + - name: T.malloc + execute: run-test /tset/ANSI.os/genuts/malloc/T.malloc + - name: T.qsort + execute: run-test /tset/ANSI.os/genuts/qsort/T.qsort + - name: T.rand + execute: run-test /tset/ANSI.os/genuts/rand/T.rand + - name: T.realloc + execute: run-test /tset/ANSI.os/genuts/realloc/T.realloc + - name: T.srand + execute: run-test /tset/ANSI.os/genuts/srand/T.srand + - name: T.longjmp + execute: run-test /tset/ANSI.os/jump/longjmp/T.longjmp + - name: T.setjmp + execute: run-test /tset/ANSI.os/jump/setjmp/T.setjmp + - name: T.acos + execute: run-test /tset/ANSI.os/maths/acos/T.acos + - name: T.asin + execute: run-test /tset/ANSI.os/maths/asin/T.asin + - name: T.atan + execute: run-test /tset/ANSI.os/maths/atan/T.atan + - name: T.atan2 + execute: run-test /tset/ANSI.os/maths/atan2/T.atan2 + - name: T.ceil + execute: run-test /tset/ANSI.os/maths/ceil/T.ceil + - name: T.cos + execute: run-test /tset/ANSI.os/maths/cos/T.cos + - name: T.cosh + execute: run-test /tset/ANSI.os/maths/cosh/T.cosh + - name: T.exp + execute: run-test /tset/ANSI.os/maths/exp/T.exp + - name: T.fabs + execute: run-test /tset/ANSI.os/maths/fabs/T.fabs + - name: T.floor + execute: run-test /tset/ANSI.os/maths/floor/T.floor + - name: T.fmod + execute: run-test /tset/ANSI.os/maths/fmod/T.fmod + - name: T.frexp + execute: run-test /tset/ANSI.os/maths/frexp/T.frexp + - name: T.ldexp + execute: run-test /tset/ANSI.os/maths/ldexp/T.ldexp + - name: T.log + execute: run-test /tset/ANSI.os/maths/log/T.log + - name: T.log10 + execute: run-test /tset/ANSI.os/maths/log10/T.log10 + - name: T.modf + execute: run-test /tset/ANSI.os/maths/modf/T.modf + - name: T.pow + execute: run-test /tset/ANSI.os/maths/pow/T.pow + - name: T.sin + execute: run-test /tset/ANSI.os/maths/sin/T.sin + - name: T.sinh + execute: run-test /tset/ANSI.os/maths/sinh/T.sinh + - name: T.sqrt + execute: run-test /tset/ANSI.os/maths/sqrt/T.sqrt + - name: T.tan + execute: run-test /tset/ANSI.os/maths/tan/T.tan + - name: tanh + execute: run-test /tset/ANSI.os/maths/tanh/T.tanh + - name: T.clearerr + execute: run-test /tset/ANSI.os/streamio/clearerr/T.clearerr + - name: T.fclose + ignore: True + execute: run-test T.fclose + - name: T.feof + execute: run-test /tset/ANSI.os/streamio/feof/T.feof + - name: T.ferror + execute: run-test /tset/ANSI.os/streamio/ferror/T.ferror + - name: T.fflush + ignore: True + execute: run-test T.fflush + - name: T.fgets + ignore: True + execute: run-test T.fgets + - name: T.fopen + ignore: True + execute: run-test T.fopen + - name: T.fputs + ignore: True + execute: run-test T.fputs + - name: T.fread + ignore: True + execute: run-test T.fread + - name: T.freopen + ignore: True + execute: run-test T.freopen + - name: T.fseek + execute: run-test /tset/ANSI.os/streamio/fseek/T.fseek + - name: T.ftell + execute: run-test /tset/ANSI.os/streamio/ftell/T.ftell + - name: T.fwrite + ignore: True + execute: run-test T.fwrite + - name: T.getc + ignore: True + execute: run-test T.getc + - name: T.gets + ignore: True + execute: run-test T.gets + - name: T.perror + execute: run-test /tset/ANSI.os/streamio/perror/T.perror + - name: T.printf + ignore: True + execute: run-test T.printf + - name: T.putc + ignore: True + execute: run-test T.putc + - name: T.puts + ignore: True + execute: run-test T.puts + - name: T.remove + ignore: True + execute: run-test T.remove + - name: T.rewind + execute: run-test /tset/ANSI.os/streamio/rewind/T.rewind + - name: T.scanf + ignore: True + execute: run-test T.scanf + - name: T.setbuf + ignore: True + execute: run-test /tset/ANSI.os/streamio/setbuf/T.setbuf + - name: T.tmpfile + ignore: True + execute: run-test /tset/ANSI.os/streamio/tmpfile/T.tmpfile + - name: T.tmpnam + ignore: True + execute: run-test /tset/ANSI.os/streamio/tmpnam/T.tmpnam + - name: T.ungetc + execute: run-test /tset/ANSI.os/streamio/ungetc/T.ungetc + - name: T.vprintf + ignore: True + execute: run-test T.vprintf + - name: T.strcat + execute: run-test /tset/ANSI.os/string/strcat/T.strcat + - name: T.strchr + execute: run-test /tset/ANSI.os/string/strchr/T.strchr + - name: T.strcmp + execute: run-test /tset/ANSI.os/string/strcmp/T.strcmp + - name: T.strcpy + execute: run-test /tset/ANSI.os/string/strcpy/T.strcpy + - name: T.strscpn + execute: run-test /tset/ANSI.os/string/strcspn/T.strcspn + - name: T.strlen + execute: run-test /tset/ANSI.os/string/strlen/T.strlen + - name: T.strncat + execute: run-test /tset/ANSI.os/string/strncat/T.strncat + - name: T.strncmp + execute: run-test /tset/ANSI.os/string/strncmp/T.strncmp + - name: T.strncpy + execute: run-test /tset/ANSI.os/string/strncpy/T.strncpy + - name: T.strpbrk + execute: run-test /tset/ANSI.os/string/strpbrk/T.strpbrk + - name: T.strrchr + execute: run-test /tset/ANSI.os/string/strrchr/T.strrchr + - name: T.strspn + execute: run-test /tset/ANSI.os/string/strspn/T.strspn + - name: T.strstr + execute: run-test /tset/ANSI.os/string/strstr/T.strstr + - name: T.strtok + execute: run-test /tset/ANSI.os/string/strtok/T.strtok + - name: T.asctime + execute: run-test /tset/ANSI.os/time/asctime/T.asctime + - name: T.ctime + ignore: True + execute: run-test /tset/ANSI.os/time/ctime/T.ctime + - name: T.gmtime + execute: run-test /tset/ANSI.os/time/gmtime/T.gmtime + - name: T.localtime + ignore: True + execute: run-test /tset/ANSI.os/time/localtime/T.localtime + - name: T.mktime + execute: run-test /tset/ANSI.os/time/mktime/T.mktime + - name: T.strftime + ignore: True + execute: run-test /tset/ANSI.os/time/strftime/T.strftime + - name: T.fdopen + execute: run-test /tset/POSIX.os/Csupport/fdopen/T.fdopen + - name: T.fileno + execute: run-test /tset/POSIX.os/Csupport/fileno/T.fileno + - name: T.siglongjmp + execute: run-test /tset/POSIX.os/Csupport/siglongjmp/T.siglongjmp + - name: T.sigsetjmp + execute: run-test /tset/POSIX.os/Csupport/sigsetjmp/T.sigsetjmp + - name: T.tzset + execute: run-test /tset/POSIX.os/Csupport/tzset/T.tzset + - name: T.cpio + ignore: True + execute: run-test T.cpio + - name: T.tar + ignore: True + execute: run-test T.tar + - name: T.T.c_cc + ignore: True + execute: run-test T.c_cc + - name: T.c_cflag + ignore: True + execute: run-test T.c_cflag + - name: T.c_iflag + ignore: True + execute: run-test T.c_iflag + - name: T.c_lflag + ignore: True + execute: run-test T.c_lflag + - name: T.c_oflag + ignore: True + execute: run-test T.c_oflag + - name: T.cfgetispee + execute: run-test /tset/POSIX.os/devclass/cfgetispee/T.cfgetispee + - name: T.cfgetospee + execute: run-test /tset/POSIX.os/devclass/cfgetospee/T.cfgetospee + - name: T.cfsetispee + ignore: True + execute: run-test T.cfsetispee + - name: T.cfsetospee + ignore: True + execute: run-test T.cfsetospee + - name: T.i_access + ignore: True + execute: run-test T.i_access + - name: T.i_canon + ignore: True + execute: run-test T.i_canon + - name: T.i_close + ignore: True + execute: run-test T.i_close + - name: T.i_ctty + ignore: True + execute: run-test T.i_ctty + - name: T.i_inproc + ignore: True + execute: run-test T.i_inproc + - name: T.i_modem + ignore: True + execute: run-test T.i_modem + - name: T.i_noncanon + ignore: True + execute: run-test T.i_noncanon + - name: T.i_spchars + ignore: True + execute: run-test T.i_spchars + - name: T.tcdrain + ignore: True + execute: run-test T.tcdrain + - name: T.tcflow + ignore: True + execute: run-test T.tcflow + - name: T.tcflush + ignore: True + execute: run-test T.tcflush + - name: T.tcgetattr + ignore: True + execute: run-test T.tcgetattr + - name: T.tcgetpgrp + ignore: True + execute: run-test T.tcgetpgrp + - name: T.tcsendbrea + ignore: True + execute: run-test T.tcsendbrea + - name: T.tcsetattr + ignore: True + execute: run-test T.tcsetattr + - name: T.tcsetpgrp + ignore: True + execute: run-test T.tcsetpgrp + - name: T.access + ignore: True + execute: run-test T.access + - name: T.chdir + ignore: True + execute: run-test T.chdir + - name: T.chmod + ignore: True + execute: run-test T.chmod + - name: T.chown + ignore: True + execute: run-test T.chown + - name: T.closedir + execute: run-test /tset/POSIX.os/files/closedir/T.closedir + - name: T.creat + ignore: True + execute: run-test T.creat + - name: T.fpathconf + ignore: True + execute: run-test T.fpathconf + - name: T.fstat + execute: run-test /tset/POSIX.os/files/fstat/T.fstat + - name: T.getcwd + execute: run-test /tset/POSIX.os/files/getcwd/T.getcwd + - name: T.link + ignore: True + execute: run-test T.link + - name: T.mkdir + ignore: True + execute: run-test T.mkdir + - name: T.mkfifo + ignore: True + execute: run-test T.mkfifo + - name: T.open + ignore: True + execute: run-test T.open + - name: T.opendir + ignore: True + execute: run-test T.opendir + - name: T.pathconf + ignore: True + execute: run-test T.pathconf + - name: T.readdir + ignore: True + execute: run-test T.readdir + - name: T.rename + ignore: True + execute: run-test T.rename + - name: T.rewinddir + execute: run-test /tset/POSIX.os/files/rewinddir/T.rewinddir + - name: T.rmdir + ignore: True + execute: run-test T.rmdir + - name: T.stat + ignore: True + execute: run-test T.stat + - name: T.umask + execute: run-test /tset/POSIX.os/files/umask/T.umask + - name: T.unlink + ignore: True + execute: run-test T.unlink + - name: T.utime + ignore: True + execute: run-test T.utime + - name: T.close + execute: run-test /tset/POSIX.os/ioprim/close/T.close + - name: T.dup + execute: run-test /tset/POSIX.os/ioprim/dup/T.dup + - name: T.dup2 + ignore: True + execute: run-test /tset/POSIX.os/ioprim/dup2/T.dup2 + - name: T.fcntl + ignore: True + execute: run-test T.fcntl + - name: T.lseek + execute: run-test /tset/POSIX.os/ioprim/lseek/T.lseek + - name: T.pipe + ignore: True + execute: run-test /tset/POSIX.os/ioprim/pipe/T.pipe + - name: T.read + ignore: True + execute: run-test T.read + - name: T.write + ignore: True + execute: run-test T.write + - name: T.ctermid + ignore: True + execute: run-test T.ctermid + - name: T.getegid + execute: run-test /tset/POSIX.os/procenv/getegid/T.getegid + - name: T.getenv + execute: run-test /tset/POSIX.os/procenv/getenv/T.getenv + - name: T.geteuid + execute: run-test /tset/POSIX.os/procenv/geteuid/T.geteuid + - name: T.getgid + execute: run-test /tset/POSIX.os/procenv/getgid/T.getgid + - name: T.getgroups + execute: run-test /tset/POSIX.os/procenv/getgroups/T.getgroups + - name: T.getlogin + execute: run-test /tset/POSIX.os/procenv/getlogin/T.getlogin + - name: T.getpgrp + execute: run-test /tset/POSIX.os/procenv/getpgrp/T.getpgrp + - name: T.getpid + execute: run-test /tset/POSIX.os/procenv/getpid/T.getpid + - name: getppid + execute: run-test /tset/POSIX.os/procenv/getppid/T.getppid + - name: T.getuid + execute: run-test /tset/POSIX.os/procenv/getuid/T.getuid + - name: T.isatty + execute: run-test /tset/POSIX.os/procenv/isatty/T.isatty + - name: T.setgid + execute: run-test /tset/POSIX.os/procenv/setgid/T.setgid + - name: setpgid + execute: run-test /tset/POSIX.os/procenv/setpgid/T.setpgid + - name: T.setsid + ignore: True + execute: run-test T.setsid + - name: T.setuid + execute: run-test /tset/POSIX.os/procenv/setuid/T.setuid + - name: T.sysconf + execute: run-test /tset/POSIX.os/procenv/sysconf/T.sysconf + - name: T.time + execute: run-test /tset/POSIX.os/procenv/time/T.time + - name: T.times + execute: run-test /tset/POSIX.os/procenv/times/T.times + - name: T.ttyname + execute: run-test /tset/POSIX.os/procenv/ttyname/T.ttyname + - name: T.uname + execute: run-test /tset/POSIX.os/procenv/uname/T.uname + - name: T._exit + ignore: True + execute: run-test T._exit + - name: T.alarm + execute: run-test /tset/POSIX.os/procprim/alarm/T.alarm + - name: T.exec + ignore: True + execute: run-test T.exec + - name: T.fork + ignore: True + execute: run-test T.fork + - name: T.kill + execute: run-test /tset/POSIX.os/procprim/kill/T.kill + - name: T.pause + execute: run-test /tset/POSIX.os/procprim/pause/T.pause + - name: T.sigaction + execute: run-test /tset/POSIX.os/procprim/sigaction/T.sigaction + - name: T.sigaddset + execute: run-test /tset/POSIX.os/procprim/sigaddset/T.sigaddset + - name: T.sigconcept + ignore: True + execute: run-test T.sigconcept + - name: T.sigdelset + execute: run-test /tset/POSIX.os/procprim/sigdelset/T.sigdelset + - name: T.sigemptyse + execute: run-test /tset/POSIX.os/procprim/sigemptyse/T.sigemptyse + - name: T.sigfillset + execute: run-test /tset/POSIX.os/procprim/sigfillset/T.sigfillset + - name: T.sigismembe + execute: run-test /tset/POSIX.os/procprim/sigismembe/T.sigismembe + - name: T.sigpending + ignore: True + execute: run-test T.sigpending + - name: T.sigprocmas + execute: run-test /tset/POSIX.os/procprim/sigprocmas/T.sigprocmas + - name: T.sigsuspend + execute: run-test /tset/POSIX.os/procprim/sigsuspend/T.sigsuspend + - name: T.sleep + execute: run-test /tset/POSIX.os/procprim/sleep/T.sleep + - name: T.wait + execute: run-test /tset/POSIX.os/procprim/wait/T.wait + - name: T.waitpid + ignore: True + execute: run-test /tset/POSIX.os/procprim/waitpid/T.waitpid + - name: T.getgrgid + execute: run-test /tset/POSIX.os/sysdb/getgrgid/T.getgrgid + - name: T.getgrnam + execute: run-test /tset/POSIX.os/sysdb/getgrnam/T.getgrnam + - name: T.getpwnam + execute: run-test /tset/POSIX.os/sysdb/getpwnam/T.getpwnam + - name: T.getpwuid + execute: run-test /tset/POSIX.os/sysdb/getpwuid/T.getpwuid