Skip to content

Commit

Permalink
Merge pull request #102 from woocommerce/23-12/self-test-output
Browse files Browse the repository at this point in the history 
Improve self-test output, allow bigger variance of debug_logs
  • Loading branch information
@Luc45
Luc45 authored Dec 20, 2023
2 parents 84c0277 + ef880c9 commit 08c6f16
Show file tree
Hide file tree
Showing 8 changed files with 89 additions and 21 deletions.
48 changes: 45 additions & 3 deletions _tests/ParallelOutput.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@
class ParallelOutput {
private $outputBuffer;
private $headers;
private $processStatus;
private $startTimes;

public function __construct() {
$this->outputBuffer = [];
$this->headers = [];
$this->outputBuffer = [];
$this->headers = [];
$this->processStatus = [];
$this->startTimes = [];
register_shutdown_function( [ $this, 'onShutdown' ] );
}

Expand All @@ -18,8 +22,9 @@ public function processOutputCallback( $out, Process $process ) {

// Check and store the header for the task
if ( ! isset( $this->headers[ $taskId ] ) ) {
$this->headers[ $taskId ] = $process->getEnv()['qit_task_id'];
$this->headers[ $taskId ] = $process->getEnv()['qit_task_id'] . "\n";
$this->outputBuffer[ $taskId ] = "";
$this->startTimes[ $taskId ] = microtime( true );
}

// Append new output to the buffer for the task
Expand All @@ -28,6 +33,31 @@ public function processOutputCallback( $out, Process $process ) {
if ( ! getenv( "CI" ) ) {
$this->displayBufferedOutputs();
}

$this->updateProcessStatus( $process, $out ); // You need to implement this method
}

protected function updateProcessStatus( Process $process, $output ) {
$taskId = $process->getEnv()['qit_task_id'];

if ( $process->isRunning() ) {
$status = "\u{23F3} Running..."; // Running emoji
} elseif ( $process->isSuccessful() ) {
$status = "\u{2705} Success"; // Check mark emoji
} else {
$status = "\u{274C} Failed"; // Cross mark emoji
}

// Format the status with time before storing it
$this->processStatus[ $taskId ] = $this->formatStatusWithTime( $taskId, $status );
}

protected function formatStatusWithTime( $taskId, $status ) {
$elapsed = microtime( true ) - $this->startTimes[ $taskId ];
$minutes = floor( $elapsed / 60 );
$seconds = str_pad( (int) ceil( $elapsed % 60 ), 2, '0', STR_PAD_LEFT );

return "[$minutes:$seconds] $taskId $status";
}

protected function displayBufferedOutputs() {
Expand All @@ -39,6 +69,12 @@ protected function displayBufferedOutputs() {
echo "\033[1;33m" . $this->headers[ $taskId ] . "\033[0m"; // Yellow for headers
echo $output . "\n";
}

// Print the summary section
echo "\n\033[1;32mSummary Section:\033[0m\n"; // Green for summary section header
foreach ( $this->processStatus as $status ) {
echo "$status\n";
}
}

public function onShutdown() {
Expand All @@ -48,6 +84,12 @@ public function onShutdown() {
echo "\033[1;33m" . $this->headers[ $taskId ] . "\033[0m\n"; // Yellow for headers
echo $output; // Print the accumulated output
}

// Print the summary section
echo "\n\033[1;32mSummary Section:\033[0m\n"; // Green for summary section header
foreach ( $this->processStatus as $status ) {
echo "$status\n";
}
}
}
}
33 changes: 21 additions & 12 deletions _tests/QITSelfTests.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,15 +174,25 @@ function generate_test_runs( array $test_types ): array {
}

function add_task_id_to_process( Process $process, array $test_run ) {
$task_id = sprintf(
"[Test Type %s [Slug %s] [PHP %s] [WP %s] [Woo %s] [Features %s])]: \n",
$test_run['type'],
$test_run['slug'],
$test_run['php'],
$test_run['wp'],
$test_run['woo'],
$test_run['features']
);
$task_id_parts = [
sprintf( "[%s -", ucwords( $test_run['type'] ) ),
sprintf( "%s]", $test_run['slug'] )
];

if ( ! empty( $test_run['php'] ) ) {
$task_id_parts[] = sprintf( "[PHP %s]", $test_run['php'] );
}
if ( ! empty( $test_run['wp'] ) ) {
$task_id_parts[] = sprintf( "[WP %s]", $test_run['wp'] );
}
if ( ! empty( $test_run['woo'] ) ) {
$task_id_parts[] = sprintf( "[Woo %s]", $test_run['woo'] );
}
if ( ! empty( $test_run['features'] ) ) {
$task_id_parts[] = sprintf( "[Features %s]", $test_run['features'] );
}

$task_id = implode( ' ', $task_id_parts ) . ": ";

$process->setEnv( array_merge( $process->getEnv(), [ 'qit_task_id' => $task_id ] ) );
}
Expand Down Expand Up @@ -397,9 +407,8 @@ function handle_qit_response( Process $qit_process, string $out, array &$failed_
// echo "[INFO] Preparing to run test: {$phpunit_process->getCommandLine()}\n";

try {
$phpunit_process->mustRun( function ( $type, $out ) use ( $phpunit_process ) {
$GLOBALS['parallelOutput']->processOutputCallback( $out, $phpunit_process );
} );
$phpunit_process->mustRun();
$GLOBALS['parallelOutput']->processOutputCallback( $phpunit_process->getOutput(), $phpunit_process );
} catch ( ProcessFailedException $e ) {
$failed_tests[] = $e;
} finally {
Expand Down
13 changes: 11 additions & 2 deletions _tests/tests/QITE2ETestCase.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,8 @@ public function validate_and_normalize( string $file_path, ?callable $callback =
* Normalization rules for 'count':
* - Exact values are retained for counts below 50.
* - Counts between 50 and 100 are rounded to the nearest 5.
* - Counts above 100 are rounded to the nearest 10.
* - Counts between 100 and 200 are rounded to the nearest 10.
* - Counts above 200 are rounded to the nearest 25.
*
* Additionally, certain known failure messages (e.g., WordPress.org connectivity issues)
* are conditionally removed from the logs.
Expand All @@ -175,14 +176,22 @@ public function validate_and_normalize( string $file_path, ?callable $callback =
$debug_log['count'] = round( $debug_log['count'] / 5 ) * 5; // Round to the closest 5 if not already divisible by 5.
echo "{$debug_log['count']}\n";
}
} else {
} elseif ( $debug_log['count'] < 200 ) {
if ( $debug_log['count'] % 10 === 0 ) {
echo "Skipping normalization as it's already divisible by 10\n";
} else {
echo "Normalizing debug_log.count from {$debug_log['count']} to ";
$debug_log['count'] = round( $debug_log['count'] / 10 ) * 10; // Round to the closest 10 if not already divisible by 10.
echo "{$debug_log['count']}\n";
}
} else {
if ( $debug_log['count'] % 25 === 0 ) {
echo "Skipping normalization as it's already divisible by 25\n";
} else {
echo "Normalizing debug_log.count from {$debug_log['count']} to ";
$debug_log['count'] = round( $debug_log['count'] / 25 ) * 25; // Round to the closest 25 if not already divisible by 25.
echo "{$debug_log['count']}\n";
}
}

// Handle E2E Delete Products tests with more wiggle room.
Expand Down
2 changes: 1 addition & 1 deletion _tests/tests/__snapshots__/ApiTest__test_api_valid_features_rc_74__1.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1381,7 +1381,7 @@
{
"debug_log": [
{
"count": "560",
"count": "550",
"message": "PHP Notice: New Product Editor is enabled as expected. in \\/var\\/www\\/html\\/wp-content\\/plugins\\/woocommerce-product-feeds\\/woocommerce-product-feeds.php on line 12"
}
]
Expand Down
2 changes: 1 addition & 1 deletion _tests/tests/__snapshots__/ApiTest__test_api_valid_features_rc_82__1.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1381,7 +1381,7 @@
{
"debug_log": [
{
"count": "560",
"count": "550",
"message": "PHP Notice: New Product Editor is enabled as expected. in \\/var\\/www\\/html\\/wp-content\\/plugins\\/woocommerce-product-feeds\\/woocommerce-product-feeds.php on line 12"
}
]
Expand Down
2 changes: 1 addition & 1 deletion _tests/tests/__snapshots__/ApiTest__test_api_valid_features_stable_74__1.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1381,7 +1381,7 @@
{
"debug_log": [
{
"count": "560",
"count": "550",
"message": "PHP Notice: New Product Editor is enabled as expected. in \\/var\\/www\\/html\\/wp-content\\/plugins\\/woocommerce-product-feeds\\/woocommerce-product-feeds.php on line 12"
}
]
Expand Down
2 changes: 1 addition & 1 deletion _tests/tests/__snapshots__/ApiTest__test_api_valid_features_stable_82__1.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1381,7 +1381,7 @@
{
"debug_log": [
{
"count": "560",
"count": "550",
"message": "PHP Notice: New Product Editor is enabled as expected. in \\/var\\/www\\/html\\/wp-content\\/plugins\\/woocommerce-product-feeds\\/woocommerce-product-feeds.php on line 12"
}
]
Expand Down
8 changes: 8 additions & 0 deletions _tests/tests/__snapshots__/E2eTest__test_e2e_delete_products_stable_74__1.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1585,6 +1585,14 @@
},
{
"debug_log": [
{
"count": "Less than 10",
"message": "PHP Notice: Function map_meta_cap was called incorrectly. When checking for the edit_post capability, you must always check it against a specific post. Please see Debugging in WordPress for more information. (This message was added in version 6.1.0.) in \\/var\\/www\\/html\\/wp-includes\\/functions.php on line 6031"
},
{
"count": "Less than 10",
"message": "PHP Notice: Trying to get property \'ID\' of non-object in \\/var\\/www\\/html\\/wp-admin\\/includes\\/post.php on line 2132"
},
{
"count": "Less than 10",
"message": "PHP Notice: Trying to get property \'post_mime_type\' of non-object in \\/var\\/www\\/html\\/wp-admin\\/includes\\/post.php on line 273"
Expand Down

0 comments on commit 08c6f16

Please sign in to comment.