mozilla-mobile · github-actions · Jan 23, 2025
@@ -6,6 +6,7 @@ const lazy = {};
 ChromeUtils.defineESModuleGetters(lazy, {
   FormAutofill: "resource://autofill/FormAutofill.sys.mjs",
   FormAutofillUtils: "resource://gre/modules/shared/FormAutofillUtils.sys.mjs",
+  MLAutofill: "resource://autofill/MLAutofill.sys.mjs",
 });
 
 /**
@@ -98,6 +99,8 @@ export class FieldDetail {
       fathomLabel = null,
       fathomConfidence = null,
       isVisible = true,
+      mlHeaderInput = null,
+      mlButtonInput = null,
     } = {}
   ) {
     const fieldDetail = new FieldDetail(element);
@@ -169,7 +172,9 @@ export class FieldDetail {
       lazy.FormAutofill.isMLExperimentEnabled &&
       ["input", "select"].includes(element.localName)
     ) {
-      fieldDetail.htmlMarkup = element.outerHTML.substring(0, 1024);
+      fieldDetail.mlinput = lazy.MLAutofill.getMLMarkup(fieldDetail.element);
+      fieldDetail.mlHeaderInput = mlHeaderInput;
+      fieldDetail.mlButtonInput = mlButtonInput;
       fieldDetail.fathomLabel = fathomLabel;
       fieldDetail.fathomConfidence = fathomConfidence;
     }

@@ -83,10 +83,22 @@ export const FormAutofill = {
     }
     return false;
   },
+
+  /**
+   * Return true if address autofill is available for a specific country.
+   */
   isAutofillAddressesAvailableInCountry(country) {
-    return FormAutofill._addressAutofillSupportedCountries.includes(
-      country.toUpperCase()
-    );
+    if (FormAutofill._isAutofillAddressesAvailableInExperiment) {
+      return true;
+    }
+
+    let available = FormAutofill._isAutofillAddressesAvailable;
+    if (country && available == "detect") {
+      return FormAutofill._addressAutofillSupportedCountries.includes(
+        country.toUpperCase()
+      );
+    }
+    return available == "on";
   },
   get isAutofillEnabled() {
     return this.isAutofillAddressesEnabled || this.isAutofillCreditCardsEnabled;
@@ -317,6 +329,13 @@ XPCOMUtils.defineLazyPreferenceGetter(
   false
 );
 
+XPCOMUtils.defineLazyPreferenceGetter(
+  FormAutofill,
+  "MLModelRevision",
+  "extensions.formautofill.ml.experiment.modelRevision",
+  null
+);
+
 ChromeUtils.defineLazyGetter(FormAutofill, "countries", () =>
   AddressMetaDataLoader.getCountries()
 );
@@ -71,7 +71,7 @@ export class FormAutofillChild {
 
     if (!handler.hasIdentifiedFields() || handler.updateFormIfNeeded(element)) {
       // If we found newly identified fields, run section classification heuristic
-      const detectedFields = FormAutofillHandler.collectFormFields(
+      const detectedFields = FormAutofillHandler.collectFormFieldDetails(
         handler.form
       );
 

@@ -217,20 +217,38 @@ export class FormAutofillHandler {
    * Collect <input>, <select>, and <iframe> elements from the specified form
    * and return the correspond 'FieldDetail' objects.
    *
-   * @param {HTMLFormElement} form
+   * @param {formLike} formLike
    *        The form that we collect information from.
+   * @param {boolean} includeIframe
+   *        True to add <iframe> to the returned FieldDetails array.
+   * @param {boolean} ignoreInvisibleInput
+   *        True to NOT run heuristics on invisible <input> fields.
    *
    * @returns {Array<FieldDeail>}
    *        An array containing eliglble fields for autofill, also
    *        including iframe.
    */
-  static collectFormFields(form) {
-    const fieldDetails = lazy.FormAutofillHeuristics.getFormInfo(form) ?? [];
+  static collectFormFieldDetails(
+    formLike,
+    includeIframe,
+    ignoreInvisibleInput = true
+  ) {
+    const fieldDetails =
+      lazy.FormAutofillHeuristics.getFormInfo(formLike, ignoreInvisibleInput) ??
+      [];
+
+    // 'FormLike' only contains <input> & <select>, so in order to include <iframe>
+    // in the list of 'FieldDetails', we need to search for <iframe> in the form.
+    if (!includeIframe) {
+      return fieldDetails;
+    }
 
-    let index = 0;
+    // Insert <iframe> elements into the fieldDetails array, maintaining the element order.
     const fieldDetailsIncludeIframe = [];
-    const elements = form.rootElement.querySelectorAll("input, select, iframe");
-
+    let index = 0;
+    const elements = formLike.rootElement.querySelectorAll(
+      "input, select, iframe"
+    );
     for (const element of elements) {
       if (fieldDetails[index]?.element == element) {
         fieldDetailsIncludeIframe.push(fieldDetails[index]);
@@ -239,8 +257,17 @@ export class FormAutofillHandler {
         element.localName == "iframe" &&
         FormAutofillUtils.isFieldVisible(element)
       ) {
-        const iframeFd = lazy.FieldDetail.create(element, form, "iframe");
-        fieldDetailsIncludeIframe.push(iframeFd);
+        // Add the <iframe> only if it is under the `formLike` element.
+        // While we use formLike.rootElement.querySelectorAll, it is still possible
+        // we find an <iframe> inside a <form> within this rootElement. In this
+        // case, we don't want to include the <iframe> in the field list.
+        if (
+          lazy.AutofillFormFactory.findRootForField(element) ==
+          formLike.rootElement
+        ) {
+          const iframeFd = lazy.FieldDetail.create(element, formLike, "iframe");
+          fieldDetailsIncludeIframe.push(iframeFd);
+        }
       }
     }
     return fieldDetailsIncludeIframe;

@@ -13,6 +13,7 @@ ChromeUtils.defineESModuleGetters(lazy, {
   FieldScanner: "resource://gre/modules/shared/FieldScanner.sys.mjs",
   FormAutofillUtils: "resource://gre/modules/shared/FormAutofillUtils.sys.mjs",
   LabelUtils: "resource://gre/modules/shared/LabelUtils.sys.mjs",
+  MLAutofill: "resource://autofill/MLAutofill.sys.mjs",
 });
 
 /**
@@ -325,6 +326,37 @@ export const FormAutofillHeuristics = {
     return savedIndex != scanner.parsingIndex;
   },
 
+  /**
+   * If this is a house number field and there is no address-line1 or
+   * street-address field, change the house number field to address-line1.
+   *
+   * @param {FieldScanner} scanner
+   *        The current parsing status for all elements
+   * @returns {boolean}
+   *          Return true if there is any field can be recognized in the parser,
+   *          otherwise false.
+   */
+  _parseHouseNumberFields(scanner, fieldDetail) {
+    if (fieldDetail?.fieldName == "address-housenumber") {
+      const savedIndex = scanner.parsingIndex;
+      for (let idx = 0; !scanner.parsingFinished; idx++) {
+        const detail = scanner.getFieldDetailByIndex(idx);
+        if (!detail) {
+          break;
+        }
+
+        if (["address-line1", "street-address"].includes(detail?.fieldName)) {
+          return false;
+        }
+      }
+
+      // Return false so additional address handling still gets performed.
+      scanner.updateFieldName(savedIndex, "street-address");
+    }
+
+    return false;
+  },
+
   /**
    * Try to find the correct address-line[1-3] sequence and correct their field
    * names.
@@ -633,6 +665,26 @@ export const FormAutofillHeuristics = {
       prevCCFields.add(detail.fieldName);
     }
 
+    const subsequentCCFields = new Set();
+
+    for (let idx = scanner.parsingIndex + fields.length; ; idx++) {
+      const detail = scanner.getFieldDetailByIndex(idx);
+      if (
+        // For updates we only check subsequent fields that are not of type address or do not have an
+        // alternative field name that is of type address, to avoid falsely updating address
+        // form name fields to cc-*-name.
+        lazy.FormAutofillUtils.getCategoryFromFieldName(detail?.fieldName) !=
+          "creditCard" ||
+        (detail?.alternativeFieldName !== undefined &&
+          lazy.FormAutofillUtils.getCategoryFromFieldName(
+            detail?.alternativeFieldName
+          ) != "creditCard")
+      ) {
+        break;
+      }
+      subsequentCCFields.add(detail.fieldName);
+    }
+
     const isLastField =
       scanner.getFieldDetailByIndex(scanner.parsingIndex + 1) === null;
 
@@ -646,11 +698,17 @@ export const FormAutofillHeuristics = {
     //    because "cc-csc" is often the last field in a credit card form, and we want to
     //    avoid mistakenly updating fields in subsequent address forms.
     if (
-      ["cc-number"].some(f => prevCCFields.has(f)) &&
-      !["cc-name", "cc-given-name", "cc-family-name"].some(f =>
-        prevCCFields.has(f)
-      ) &&
-      (isLastField || !prevCCFields.has("cc-csc"))
+      (["cc-number"].some(f => prevCCFields.has(f)) &&
+        !["cc-name", "cc-given-name", "cc-family-name"].some(f =>
+          prevCCFields.has(f)
+        ) &&
+        (isLastField || !prevCCFields.has("cc-csc"))) || // 4. Or we update when current name field is followed by
+      //    creditcard form fields that contain cc-number
+      //    and no cc-*-name field is detected
+      (["cc-number"].some(f => subsequentCCFields.has(f)) &&
+        !["cc-name", "cc-given-name", "cc-family-name"].some(f =>
+          subsequentCCFields.has(f)
+        ))
     ) {
       // If there is only one field, assume the name field a `cc-name` field
       if (fields.length == 1) {
@@ -704,23 +762,37 @@ export const FormAutofillHeuristics = {
    * in the belonging section. The details contain the autocomplete info
    * (e.g. fieldName, section, etc).
    *
-   * @param {HTMLFormElement} form
+   * @param {formLike} formLike
    *        the elements in this form to be predicted the field info.
+   * @param {boolean} ignoreInvisibleInput
+   *        True to NOT run heuristics on invisible <input> fields.
    * @returns {Array<FormSection>}
    *        all sections within its field details in the form.
    */
-  getFormInfo(form) {
-    const elements = Array.from(form.elements).filter(element =>
+  getFormInfo(formLike, ignoreInvisibleInput) {
+    const elements = Array.from(formLike.elements).filter(element =>
       lazy.FormAutofillUtils.isCreditCardOrAddressFieldType(element)
     );
 
+    let closestHeaders;
+    let closestButtons;
+    if (FormAutofill.isMLExperimentEnabled && elements.length) {
+      closestHeaders = lazy.MLAutofill.closestHeaderAbove(elements);
+      closestButtons = lazy.MLAutofill.closestButtonBelow(elements);
+    }
+
     const fieldDetails = [];
-    for (const element of elements) {
+    for (let idx = 0; idx < elements.length; idx++) {
+      const element = elements[idx];
       // Ignore invisible <input>, we still keep invisible <select> since
       // some websites implements their custom dropdown and use invisible <select>
       // to store the value.
       const isVisible = lazy.FormAutofillUtils.isFieldVisible(element);
-      if (!HTMLSelectElement.isInstance(element) && !isVisible) {
+      if (
+        !HTMLSelectElement.isInstance(element) &&
+        !isVisible &&
+        ignoreInvisibleInput
+      ) {
         continue;
       }
 
@@ -742,11 +814,13 @@ export const FormAutofillHeuristics = {
       }
 
       fieldDetails.push(
-        lazy.FieldDetail.create(element, form, fieldName, {
+        lazy.FieldDetail.create(element, formLike, fieldName, {
           autocompleteInfo: inferInfo.autocompleteInfo,
           fathomLabel: inferInfo.fathomLabel,
           fathomConfidence: inferInfo.fathomConfidence,
           isVisible,
+          mlHeaderInput: closestHeaders?.[idx] ?? null,
+          mlButtonInput: closestButtons?.[idx] ?? null,
         })
       );
     }
@@ -812,6 +886,7 @@ export const FormAutofillHeuristics = {
       // Attempt to parse the field using different parsers.
       if (
         this._parseNameFields(scanner, fieldDetail) ||
+        this._parseHouseNumberFields(scanner, fieldDetail) ||
         this._parseStreetAddressFields(scanner, fieldDetail) ||
         this._parseAddressFields(scanner, fieldDetail) ||
         this._parseCreditCardExpiryFields(scanner, fieldDetail) ||