Unable to receive POST data unless I refresh client. why? How can I wait for post data to be received before page loads?

My app starts with a simple html form. the inputs are PIN# and Date Of Birth.

My express server runs on the same port 3000, when the user submits their data, puppeteer starts and logs into a specific webpage. Then I scrape the image on that webpage. Google Api takes the text from that image and saves it in an array. I then post that array string to src/results.html. But as soon as the user hits submit, they are redirected to /resuts route right immediately and the page says cannot post the data. but when I see in the console (roughly a minute later) that the post was successful, I refresh the page and I get the array of text I wanted to see.

How can I await for the data to finish being posted to the route before the page loads the data? Im using react for client side. below is my server side code. client side is just a basic react page login and a static /results page meant for the data.

const puppeteer = require("puppeteer");
const express = require("express");
const app = express();
const morgan = require("morgan");
const fs = require("fs");
const cors = require("cors");
const request = require("request-promise-native").defaults({ Jar: true });
const poll = require("promise-poller").default;

app.use(morgan("combined"));
const port = 3000;
// Imports the Google Cloud client library
const vision = require("@google-cloud/vision");
require("dotenv").config();

app.use(cors());

const textArray = [];

const App = (pinNum, dateOfB) => {
  const config = {
    sitekey: process.env.SITEKEY,
    pageurl: process.env.PAGEURL,
    apiKey: process.env.APIKEY,
    apiSubmitUrl: "http://2captcha.com/in.php",
    apiRetrieveUrl: "http://2captcha.com/res.php",
  };

  const chromeOptions = {
    executablePath: "/Program Files/Google/Chrome/Application/chrome.exe",
    headless: true,
    slowMo: 60,
    defaultViewport: null,
  };

  async function main() {
    const browser = await puppeteer.launch(chromeOptions);
    const page = await browser.newPage();

    console.log(`Navigating to ${config.pageurl}`);
    await page.goto(config.pageurl);
    try {
      const requestId = await initiateCaptchaRequest(config.apiKey);

      // const pin = getPIN();
      console.log(`Typing PIN ${pinNum}`);
      await page.type("#PIN", pinNum);

      // const dob = getDOB();
      console.log(`Typing DOB ${dateOfB}`);
      const input = await page.$("#DOB");
      await input.click({ clickCount: 3 });
      await input.type(dateOfB);

      const response = await pollForRequestResults(config.apiKey, requestId);

      console.log(`Entering recaptcha response ${response}`);
      await page.evaluate(
        `document.getElementById("g-recaptcha-response").innerHTML="${response}";`
      );

      console.log(`Submitting....`);
      page.click("#Submit");
    } catch (error) {
      console.log(
        "Your request could not be completed at this time, please check your pin number and date of birth.  Also make sure your internet connection is working and try again."
      );
      console.error(error);
    }

    await page.waitForSelector(
      "body > div.container.body-content > div:nth-child(1) > div:nth-child(2) > p"
    );
    const image = await page.$(
      "body > div.container.body-content > div:nth-child(1) > div:nth-child(2) > p"
    );
    await image.screenshot({
      path: "testResults.png",
    });

    await getImageText();
    await page.close(); // Close the website
    await browser.close(); //close browser
    await deleteImage();
  }
  main();

  //This section grabs the text off the image that was gathered from the web scraper.

  async function getImageText() {
    // Creates a client
    const client = new vision.ImageAnnotatorClient();
    console.log(`Looking for text in image`);
    // Performs label detection on the image file
    const [result] = await client.textDetection("./testResults.png");
    const [annotation] = result.textAnnotations;
    const text = annotation ? annotation.description : "";
    console.log("Extracted text from image:", text);

    //Pushed the text into a globally available array.
    textArray.push(text);

    //Sent a NOTIFICATION ALERT to the client with the text gathered from the image.
    var axios = require("axios");
    var data = JSON.stringify({
      to: "dp8vGNkcYKb-k-72j7t4Mo:APA91bEfrI3_ht89t5X1f3_Y_DACZc9DbWI4VzcYehaQoXtD_IHIFSwm9H1hgXHNq46BQwDTlCKzkWNAHbBGauEXZNQtvhQc8glz4sHQr3JY3KM7OkUEcNB7qMMpCPxRe5GzzHbe3rkE",
      notification: {
        body: text,
        title: "AverHealth Schedule",
      },
    });

    var config = {
      method: "post",
      url: "https://fcm.googleapis.com/fcm/send",
      headers: {
        "Content-Type": "application/json",
        Authorization: `key=${process.env.FCM_SERVER_KEY}`,
      },
      data: data,
    };

    axios(config)
      .then(function (response) {
        console.log(JSON.stringify(response.data));
      })
      .catch(function (error) {
        console.log(error);
      });

    
   
  }

  //Captcha Solver for the web scraper
  async function initiateCaptchaRequest(apiKey) {
    const formData = {
      key: apiKey,
      method: "userrecaptcha",
      googlekey: config.sitekey,
      json: 1,
      pageurl: config.pageurl,
    };

    console.log(
      `Submitting recaptcha request to 2captcha for ${config.pageurl}`
    );
    const response = await request.post(config.apiSubmitUrl, {
      form: formData,
    });
    console.log(response);
    return JSON.parse(response).request;
  }

  async function pollForRequestResults(
    key,
    id,
    retries = 90,
    interval = 5000,
    delay = 1500
  ) {
    console.log(`Waiting for ${delay} milliseconds....`);
    await timeout(delay);
    return poll({
      taskFn: requestCaptchaResults(key, id),
      interval,
      retries,
    });
  }

  function requestCaptchaResults(apiKey, requestId) {
    const url = `${config.apiRetrieveUrl}?key=${apiKey}&action=get&id=${requestId}&json=1`;
    console.log(url);
    return async function () {
      return new Promise(async function (resolve, reject) {
        console.log(`Polling for response...`);
        const rawResponse = await request.get(url);
        console.log(rawResponse);
        const resp = JSON.parse(rawResponse);
        console.log(resp);
        if (resp.status === 0) return reject(resp.request);
        console.log("Response received");
        console.log(resp);
        resolve(resp.request);
      });
    };
  }

  // DELETES THE FILE CREATED BY GOOGLEAPI
  function deleteImage() {
    const path = "./testResults.png";
    try {
      fs.unlinkSync(path);
      console.log("File removed:", path);
    } catch (err) {
      console.error(err);
    }
  }

  const timeout = (ms) => new Promise((res) => setTimeout(res, ms));
};

app.use(express.urlencoded({ extended: false }));

// Route to results Page
app.get("/results", (req, res) => {
  res.sendFile(__dirname + "/src/results.html");
  res.send(textArray);
});

app.post("/results", (req, res) => {
  // Insert Login Code Here
  let username = req.body.username;
  let password = req.body.password;
  App(username, password);
});

app.listen(port, () => {
  console.log(`Scraper app listening at http://localhost:${port}`);
});

Answer

I think I got the problem.

In the react app, maybe you are not using e.preventDefault() when you click submit. The browser, by default, redirects to a page where the form action is directing, if the action attribute is empty then the browser reloads the same page. I would recommend you to use e.preventDefault() on form submission and then use fetch API to make the request.

In the express server, on the route POST “results”, you are not sending any response back to the user. You should always send a response to the user. In your case you are calling the App function – which has many async functions, but you are not awaiting for App() to complete in the POST route, express is sending default response to the user as soon as it parses App() – it is not waiting for the App() to complete – express will get to this later.

You can make the (req, res) => { ... } function in the route as async function async (req, res) => { ... }, then you can make the App as async function as well. Then you can await App(...) in the route function. Also, you need to await for the main() function as well inside the App() function. Then once App() call has finished, you can send redirect response to the user.