From e48051cd0f54795bade3d3bcaf107b9c4b640cfb Mon Sep 17 00:00:00 2001 From: Michael Bodnarchuk Date: Wed, 19 Jun 2024 10:36:12 +0300 Subject: [PATCH] AI: Generate PageObject, added types, shell improvement (#4319) * added types for ai * fix: types complaint * added gen page objects, improved types, improved shell * fixed tests --------- Co-authored-by: kobenguyent --- docs/ai.md | 107 ++++++++++++++++++++++++++++++++++++++++++-- docs/pageobjects.md | 2 + lib/ai.js | 48 +++++++++++++++++++- lib/helper/AI.js | 91 ++++++++++++++++++++++++++++++++++--- lib/history.js | 19 ++++++-- lib/pause.js | 20 +++++++-- typings/index.d.ts | 61 +++++++++++++++++-------- 7 files changed, 311 insertions(+), 37 deletions(-) diff --git a/docs/ai.md b/docs/ai.md index 3f51f2ff8..a06c3b243 100644 --- a/docs/ai.md +++ b/docs/ai.md @@ -23,6 +23,7 @@ So, instead of asking "write me a test" it can ask "write a test for **this** pa CodeceptJS AI can do the following: * 🏋️‍♀️ **assist writing tests** in `pause()` or interactive shell mode +* 📃 **generate page objects** in `pause()` or interactive shell mode * 🚑 **self-heal failing tests** (can be used on CI) * 💬 send arbitrary prompts to AI provider from any tested page attaching its HTML contents @@ -260,15 +261,29 @@ By evaluating this information you will be able to check how effective AI can be ### Arbitrary GPT Prompts -What if you want to take ChatGPT on the journey of test automation and ask it questions while browsing pages? +What if you want to take AI on the journey of test automation and ask it questions while browsing pages? -This is possible with the new `AI` helper. Enable it in your config and it will automatically attach to Playwright, WebDriver, or another web helper you use. It includes the following methods: +This is possible with the new `AI` helper. Enable it in your config file in `helpers` section: + +```js +// inside codecept.conf +helpers: { + // Playwright, Puppeteer, or WebDrver helper should be enabled too + Playwright: { + }, + + AI: {} +} +``` + +AI helper will be automatically attached to Playwright, WebDriver, or another web helper you use. It includes the following methods: * `askGptOnPage` - sends GPT prompt attaching the HTML of the page. Large pages will be split into chunks, according to `chunkSize` config. You will receive responses for all chunks. * `askGptOnPageFragment` - sends GPT prompt attaching the HTML of the specific element. This method is recommended over `askGptOnPage` as you can reduce the amount of data to be processed. * `askGptGeneralPrompt` - sends GPT prompt without HTML. +* `askForPageObject` - creates PageObject for you, explained in next section. -OpenAI helper won't remove non-interactive elements, so it is recommended to manually control the size of the sent HTML. +`askGpt` methods won't remove non-interactive elements, so it is recommended to manually control the size of the sent HTML. Here are some good use cases for this helper: @@ -282,7 +297,84 @@ Here are some good use cases for this helper: const pageDoc = await I.askGptOnPageFragment('Act as technical writer, describe what is this page for', '#container'); ``` -As of now, those use cases do not apply to test automation but maybe you can apply them to your testing setup. +As of now, those use cases do not apply to test automation but maybe you can apply them to your testing setup. + +## Generate PageObjects + +Last but not the least. AI helper can be used to quickly prototype PageObjects on pages browsed within interactive session. + +![](/img/ai_page_object.png) + +Enable AI helper as explained in previous section and launch shell: + +``` +npx codeceptjs shell --ai +``` + +Also this is availble from `pause()` if AI helper is enabled, + +Ensure that browser is started in window mode, then browse the web pages on your site. +On a page you want to create PageObject execute `askForPageObject()` command. The only required parameter is the name of a page: + +```js +I.askForPageObject('login') +``` + +This command sends request to AI provider should create valid CodeceptJS PageObject. +Run it few times or switch AI provider if response is not satisfactory to you. + +> You can change the style of PageObject and locator preferences by adjusting prompt in a config file + +When completed successfully, page object is saved to **output** directory and loaded into the shell as `page` variable so locators and methods can be checked on the fly. + +If page object has `signInButton` locator you can quickly check it by typing: + +```js +I.click(page.signInButton) +``` + +If page object has `clickForgotPassword` method you can execute it as: + +```js +=> page.clickForgotPassword() +``` + +```shell +Page object for login is saved to .../output/loginPage-1718579784751.js +Page object registered for this session as `page` variable +Use `=>page.methodName()` in shell to run methods of page object +Use `click(page.locatorName)` to check locators of page object + + I.=>page.clickSignUp() + I.click(page.signUpLink) + I.=> page.enterPassword('asdasd') + I.=> page.clickSignIn() +``` + +You can improve prompt by passing custom request as a second parameter: + +```js +I.askForPageObject('login', 'implement signIn(username, password) method') +``` + +To generate page object for the part of a page, pass in root locator as third parameter. + +```js +I.askForPageObject('login', '', '#auth') +``` + +In this case, all generated locators, will use `#auth` as their root element. + +Don't aim for perfect PageObjects but find a good enough one, which you can use for writing your tests. +All created page objects are considered temporary, that's why saved to `output` directory. + +Rename created PageObject to remove timestamp and move it from `output` to `pages` folder and include it into codecept.conf file: + +```js + include: { + loginPage: "./pages/loginPage.js", + // ... +``` ## Advanced Configuration @@ -315,6 +407,7 @@ ai: { prompts: { writeStep: (html, input) => [{ role: 'user', content: 'As a test engineer...' }] healStep: (html, { step, error, prevSteps }) => [{ role: 'user', content: 'As a test engineer...' }] + generatePageObject: (html, extraPrompt = '', rootLocator = null) => [{ role: 'user', content: 'As a test engineer...' }] } } ``` @@ -392,3 +485,9 @@ To debug AI features run tests with `DEBUG="codeceptjs:ai"` flag. This will prin ``` DEBUG="codeceptjs:ai" npx codeceptjs run --ai ``` + +or if you run it in shell mode: + +``` +DEBUG="codeceptjs:ai" npx codeceptjs shell --ai +``` \ No newline at end of file diff --git a/docs/pageobjects.md b/docs/pageobjects.md index adacb85f1..c0c825d2c 100644 --- a/docs/pageobjects.md +++ b/docs/pageobjects.md @@ -56,6 +56,8 @@ module.exports = function() { ## PageObject +> ✨ CodeceptJS can [generate PageObjects using AI](/ai#generate-pageobjects). It fetches all interactive elements from a page, generates locators and methods page and writes JS code. Generated page object can be tested on the fly within the same browser session. + If an application has different pages (login, admin, etc) you should use a page object. CodeceptJS can generate a template for it with the following command: diff --git a/lib/ai.js b/lib/ai.js index 6ae5f16b9..86dffcd3b 100644 --- a/lib/ai.js +++ b/lib/ai.js @@ -31,6 +31,36 @@ const defaultPrompts = { Here is HTML code of a page where the failure has happened: \n\n${html}`, }]; }, + + generatePageObject: (html, extraPrompt = '', rootLocator = null) => [{ + role: 'user', + content: `As a test automation engineer I am creating a Page Object for a web application using CodeceptJS. + Here is an sample page object: + +const { I } = inject(); + +module.exports = { + + // setting locators + element1: '#selector', + element2: '.selector', + element3: locate().withText('text'), + + // seting methods + doSomethingOnPage(params) { + // ... + }, +} + + I want to generate a Page Object for the page I provide. + Write JavaScript code in similar manner to list all locators on the page. + Use locators in order of preference: by text (use locate().withText()), label, CSS, XPath. + Avoid TailwindCSS, Bootstrap or React style formatting classes in locators. + Add methods to to interact with page when needed. + ${extraPrompt} + ${rootLocator ? `All provided elements are inside '${rootLocator}'. Declare it as root variable and for every locator use locate(...).inside(root)` : ''} + Add only locators from this HTML: \n\n${html}`, + }], }; class AiAssistant { @@ -182,10 +212,26 @@ class AiAssistant { return this.config.response(response); } + /** + * + * @param {*} extraPrompt + * @param {*} locator + * @returns + */ + async generatePageObject(extraPrompt = null, locator = null) { + if (!this.isEnabled) return []; + if (!this.minifiedHtml) throw new Error('No HTML context provided'); + + const response = await this.createCompletion(this.prompts.generatePageObject(this.minifiedHtml, locator, extraPrompt)); + if (!response) return []; + + return this.config.response(response); + } + calculateTokens(messages) { // we implement naive approach for calculating tokens with no extra requests // this approach was tested via https://platform.openai.com/tokenizer - // we need it to display current usage tokens usage so users could analyze effectiveness of AI + // we need it to display current tokens usage so users could analyze effectiveness of AI const inputString = messages.map(m => m.content).join(' ').trim(); const numWords = (inputString.match(/[^\s\-:=]+/g) || []).length; diff --git a/lib/helper/AI.js b/lib/helper/AI.js index 0eaa8e3a1..b00d33e67 100644 --- a/lib/helper/AI.js +++ b/lib/helper/AI.js @@ -1,8 +1,14 @@ const Helper = require('@codeceptjs/helper'); +const ora = require('ora-classic'); +const fs = require('fs'); +const path = require('path'); const ai = require('../ai'); const standardActingHelpers = require('../plugin/standardActingHelpers'); const Container = require('../container'); const { splitByChunks, minifyHtml } = require('../html'); +const { beautify } = require('../utils'); +const output = require('../output'); +const { registerVariable } = require('../pause'); /** * AI Helper for CodeceptJS. @@ -10,6 +16,8 @@ const { splitByChunks, minifyHtml } = require('../html'); * This helper class provides integration with the AI GPT-3.5 or 4 language model for generating responses to questions or prompts within the context of web pages. It allows you to interact with the GPT-3.5 model to obtain intelligent responses based on HTML fragments or general prompts. * This helper should be enabled with any web helpers like Playwright or Puppeteer or WebDrvier to ensure the HTML context is available. * + * Use it only in development mode. It is recommended to run it only inside pause() mode. + * * ## Configuration * * This helper should be configured in codecept.json or codecept.conf.js @@ -66,9 +74,9 @@ class AI extends Helper { if (htmlChunks.length > 1) messages.push({ role: 'user', content: 'If action is not possible on this page, do not propose anything, I will send another HTML fragment' }); - const response = await this.aiAssistant.createCompletion(messages); + const response = await this._processAIRequest(messages); - console.log(response); + output.print(response); responses.push(response); } @@ -96,15 +104,15 @@ class AI extends Helper { { role: 'user', content: `Within this HTML: ${minifyHtml(html)}` }, ]; - const response = await this.aiAssistant.createCompletion(messages); + const response = await this._processAIRequest(messages); - console.log(response); + output.print(response); return response; } /** - * Send a general request to ChatGPT and return response. + * Send a general request to AI and return response. * @param {string} prompt * @returns {Promise} - A Promise that resolves to the generated response from the GPT model. */ @@ -113,10 +121,79 @@ class AI extends Helper { { role: 'user', content: prompt }, ]; - const response = await this.aiAssistant.createCompletion(messages); + const response = await this._processAIRequest(messages); + + output.print(response); + + return response; + } + + /** + * Generates PageObject for current page using AI. + * + * It saves the PageObject to the output directory. You can review the page object and adjust it as needed and move to pages directory. + * Prompt can be customized in a global config file. + * + * ```js + * // create page object for whole page + * I.askForPageObject('home'); + * + * // create page object with extra prompt + * I.askForPageObject('home', 'implement signIn(username, password) method'); + * + * // create page object for a specific element + * I.askForPageObject('home', null, '.detail'); + * ``` + * + * Asks for a page object based on the provided page name, locator, and extra prompt. + * + * @async + * @param {string} pageName - The name of the page to retrieve the object for. + * @param {string|null} [extraPrompt=null] - An optional extra prompt for additional context or information. + * @param {string|null} [locator=null] - An optional locator to find a specific element on the page. + * @returns {Promise} A promise that resolves to the requested page object. + */ + async askForPageObject(pageName, extraPrompt = null, locator = null) { + const html = locator ? await this.helper.grabHTMLFrom(locator) : await this.helper.grabSource(); + + const spinner = ora(' Processing AI request...').start(); + await this.aiAssistant.setHtmlContext(html); + const response = await this.aiAssistant.generatePageObject(extraPrompt, locator); + spinner.stop(); + + if (!response[0]) { + output.error('No response from AI'); + return ''; + } + + const code = beautify(response[0]); - console.log(response); + output.print('----- Generated PageObject ----'); + output.print(code); + output.print('-------------------------------'); + const fileName = path.join(output_dir, `${pageName}Page-${Date.now()}.js`); + + output.print(output.styles.bold(`Page object for ${pageName} is saved to ${output.styles.bold(fileName)}`)); + fs.writeFileSync(fileName, code); + + try { + registerVariable('page', require(fileName)); + output.success('Page object registered for this session as `page` variable'); + output.print('Use `=>page.methodName()` in shell to run methods of page object'); + output.print('Use `click(page.locatorName)` to check locators of page object'); + } catch (err) { + output.error('Error while registering page object'); + output.error(err.message); + } + + return code; + } + + async _processAIRequest(messages) { + const spinner = ora(' Processing AI request...').start(); + const response = await this.aiAssistant.createCompletion(messages); + spinner.stop(); return response; } } diff --git a/lib/history.js b/lib/history.js index 4027abc2d..01b3dc56a 100644 --- a/lib/history.js +++ b/lib/history.js @@ -10,6 +10,9 @@ const output = require('./output'); */ class ReplHistory { constructor() { + if (global.output_dir) { + this.historyFile = path.join(global.output_dir, 'cli-history'); + } this.commands = []; } @@ -21,16 +24,26 @@ class ReplHistory { this.commands.pop(); } + load() { + if (!this.historyFile) return; + if (!fs.existsSync(this.historyFile)) { + return; + } + + const history = fs.readFileSync(this.historyFile, 'utf-8'); + return history.split('\n').reverse().filter(line => line.startsWith('I.')).map(line => line.slice(2)); + } + save() { + if (!this.historyFile) return; if (this.commands.length === 0) { return; } - const historyFile = path.join(global.output_dir, 'cli-history'); const commandSnippet = `\n\n<<< Recorded commands on ${new Date()}\n${this.commands.join('\n')}`; - fs.appendFileSync(historyFile, commandSnippet); + fs.appendFileSync(this.historyFile, commandSnippet); - output.print(colors.yellow(` Commands have been saved to ${historyFile}`)); + output.print(colors.yellow(` Commands have been saved to ${this.historyFile}`)); this.commands = []; } diff --git a/lib/pause.js b/lib/pause.js index 196d73995..8e6fb7f04 100644 --- a/lib/pause.js +++ b/lib/pause.js @@ -56,7 +56,15 @@ function pauseSession(passedObject = {}) { output.print(colors.blue(' Ideas: ask it to fill forms for you or to click')); } } - rl = readline.createInterface(process.stdin, process.stdout, completer); + + rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + terminal: true, + completer, + history: history.load(), + historySize: 50, // Adjust the history size as needed + }); rl.on('line', parseInput); rl.on('close', () => { @@ -105,7 +113,7 @@ async function parseInput(cmd) { if (cmd.trim().startsWith('=>')) { isCustomCommand = true; cmd = cmd.trim().substring(2, cmd.length); - } else if (aiAssistant.isEnabled && !cmd.match(/^\w+\(/) && cmd.includes(' ')) { + } else if (aiAssistant.isEnabled && cmd.trim() && !cmd.match(/^\w+\(/) && cmd.includes(' ')) { const currentOutputLevel = output.level(); output.level(0); const res = I.grabSource(); @@ -121,7 +129,7 @@ async function parseInput(cmd) { output.level(currentOutputLevel); } - const spinner = ora("Processing OpenAI request...").start(); + const spinner = ora("Processing AI request...").start(); cmd = await aiAssistant.writeSteps(cmd); spinner.stop(); output.print(''); @@ -200,4 +208,10 @@ function completer(line) { return [hits && hits.length ? hits : completions, line]; } +function registerVariable(name, value) { + registeredVariables[name] = value; +} + module.exports = pause; + +module.exports.registerVariable = registerVariable; diff --git a/typings/index.d.ts b/typings/index.d.ts index 03ef854a0..89523177a 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -44,6 +44,44 @@ declare namespace CodeceptJS { Scenario: number; }; + type AiPrompt = { + role: string; + content: string; + } + + type AiConfig = { + /** request function to send prompts to AI provider */ + request: (messages: any) => Promise, + + /** custom prompts */ + prompts?: { + /** Returns prompt to write CodeceptJS steps inside pause mode */ + writeStep?: (html: string, input: string) => Array; + /** Returns prompt to heal step when test fails on CI if healing is on */ + healStep?: (html: string, object) => Array; + /** Returns prompt to generate page object inside pause mode */ + generatePageObject?: (html: string, extraPrompt?: string, rootLocator?: string) => Array; + }, + + /** max tokens to use */ + maxTokens?: number, + + + /** configuration for processing HTML for GPT */ + html?: { + /** max size of HTML to be sent to OpenAI to avoid token limit */ + maxLength?: number, + /** should HTML be changed by removing non-interactive elements */ + simplify?: boolean, + /** should HTML be minified before sending */ + minify?: boolean, + interactiveElements?: Array, + textElements?: Array, + allowedAttrs?: Array, + allowedRoles?: Array, + } + } + type MainConfig = { /** Pattern to locate CodeceptJS tests. * Allows to enter glob pattern or an Array of patterns to match tests / test file names. @@ -165,6 +203,9 @@ declare namespace CodeceptJS { */ JSONResponse?: any; + /** Enable AI features for development purposes */ + AI?: any; + [key: string]: any; }, /** @@ -353,25 +394,7 @@ declare namespace CodeceptJS { /** * [AI](https://codecept.io/ai/) features configuration. */ - ai?: { - /** OpenAI model to use */ - model?: string, - /** temperature, measure of randomness. Lower is better. */ - temperature?: number, - /** configuration for processing HTML for GPT */ - html?: { - /** max size of HTML to be sent to OpenAI to avoid token limit */ - maxLength?: number, - /** should HTML be changed by removing non-interactive elements */ - simplify?: boolean, - /** should HTML be minified before sending */ - minify?: boolean, - interactiveElements?: Array, - textElements?: Array, - allowedAttrs?: Array, - allowedRoles?: Array, - } - }, + ai?: AiConfig, /** * Enable full promise-based helper methods for [TypeScript](https://codecept.io/typescript/) project.