fix: improve type detection & display loader while checking values

src/utils/index.js

@@ -66,31 +66,45 @@ export function allowedStatus2change(user, isModerate, userStatus, isOwnFeature,
   return [];
 }
 
+/**
+ * Determines the type of a property based on its value.
+ *
+ * This function inspects a given property and returns a string indicating its type,
+ * such as 'boolean', 'integer', 'decimal', 'date', 'text', or 'char'.
+ * It uses various checks to determine the appropriate type for different value formats.
+ *
+ * @param {any} prop - The property value to be evaluated.
+ * @returns {string} The determined type of the property ('boolean', 'integer', 'decimal', 'date', 'text', or 'char').
+ */
 export function transformProperties(prop) {
   const type = typeof prop;
-  const date = new Date(prop);
-  const regInteger = /^-*?\d+$/;
-  const regFloat = /^-*?\d*?\.\d+$/;
-  const regText = /[\r\n]/;
-  if (type === 'boolean' || (type === 'string' && (prop.toLowerCase() === 'true' || prop.toLowerCase() === 'False'))) {
+  const regInteger = /^-?\d+$/;  // Regular expression to match integer numbers
+  const regFloat = /^-?\d*\.\d+$/;  // Regular expression to match decimal numbers
+  const regText = /[\r\n]/;  // Regular expression to detect multiline text (newlines)
+  const regDate = /^\d{4}-\d{2}-\d{2}$|^\d{2}\/\d{2}\/\d{4}$/; // Regular expression to match common date formats
+
+  // Check if the property is a boolean or a string that represents a boolean
+  if (type === 'boolean' || (type === 'string' && (prop.toLowerCase() === 'true' || prop.toLowerCase() === 'false'))) {
     return 'boolean';
-  } else if ((regInteger.test(prop) || Number.isSafeInteger(prop)) && prop !== 0) { // in case the value is 0, since it can be either float or integer, do not set to integer https://redmine.neogeo.fr/issues/16934
+  } else if (regInteger.test(prop) || (type === 'number' && Number.isSafeInteger(prop))) {
+    // Check if the property is an integer or a string that represents an integer
     return 'integer';
-  } else if (
-    type === 'string' &&
-    ['/', ':', '-'].some((el) => prop.includes(el)) && // check for chars found in datestring
-    date instanceof Date &&
-    !isNaN(date.valueOf())
-  ) {
+  } else if (type === 'string' && regDate.test(prop.trim())) {
+    // More specific check for date strings using regular expressions
     return 'date';
-  } else if (regFloat.test(prop) ||  type === 'number' && !isNaN(parseFloat(prop)) || prop === 0) { // in case the value is 0, since it can be either float or integer, by default set as a float
+  } else if (regFloat.test(prop) || (type === 'number' && !Number.isSafeInteger(prop))) {
+    // Check if the property is a decimal number or a string that represents a decimal
     return 'decimal';
-  } else if (regText.test(prop)) {
+  } else if (regText.test(prop) || (type === 'string' && prop.length > 255)) {
+    // Check if the property contains newline characters or is a long text
     return 'text';
   }
-  return 'char'; //* string by default, most accepted type in database
+
+  // Default case for all other types: assume it is a short text or character field
+  return 'char';
 }
 
+
 export function objIsEmpty(obj) {
   for(const prop in obj) {
     if(Object.hasOwn(obj, prop)) {

src/views/FeatureType/FeatureTypeDetail.vue

@@ -593,8 +593,21 @@ export default {
       return this.selectedPrerecordedListValues[listName].some((el) => el.label === fieldLabel);        
     },
 
+    /**
+     * Validates the imported data against the pre-determined field types.
+     *
+     * This function iterates over all imported features and checks if each property's value matches
+     * the expected type specified in the feature type schema. It accommodates specific type conversions,
+     * such as allowing numerical strings for 'char' or 'text' fields and converting string representations
+     * of booleans and lists as necessary.
+     *
+     * @param {Array} features - The array of imported features to validate.
+     * @returns {boolean} Returns true if all features pass the validation; otherwise, false with an error message.
+     */
     async isValidTypes(features) {
       this.importError = '';
+      
+      // Extract relevant field type information from the feature type schema
       const fields = this.feature_type.customfield_set.map((el) => {
         return {
           name: el.name,
@@ -602,32 +615,39 @@ export default {
           options: el.options,
         };
       });
-
+      let count = 1;
       for (const feature of features) {
+        this.$store.commit('DISPLAY_LOADER', `Vérification du signalement ${count} sur ${features.length}`);
         for (const { name, field_type, options } of fields) {
-          
           const properties = feature.properties || feature;
+          
           if (name in properties) {
             let fieldInFeature = properties[name];
-            if (field_type === 'boolean') { // in csv booleans are capitalized string that need to be translated in javascript syntax
+
+            // Convert boolean strings from CSV to actual booleans
+            if (field_type === 'boolean') {
               fieldInFeature = fieldInFeature === 'True' ? true : (fieldInFeature === 'False' ? false : fieldInFeature);
             }
+
             const customType = transformProperties(fieldInFeature);
-            //* if custom field value is not defined or not null or in case of prerecorded list with defined value => then check validity of field type
+
+            // Validate field only if it has a non-null, non-empty, defined value
             if (fieldInFeature !== null && fieldInFeature !== '' && fieldInFeature !== undefined) {
-              //* if field type is list, it's not possible to guess from value type
+              
+              // Handle 'list' type by checking if value is among the defined options
               if (field_type === 'list') {
-                //*then check if the value is an available option
                 if (!options.includes(fieldInFeature)) {
                   this.importError = `Fichier invalide: La valeur "${fieldInFeature}" n'est pas une option valide dans le champ "${name}" du signalement "${properties.title}".`;
+                  this.$store.commit('DISCARD_LOADER');
                   return false;
                 }
+
+              // Handle 'pre_recorded_list' by checking if the value matches pre-recorded options
               } else if (field_type === 'pre_recorded_list') {
                 if (typeof fieldInFeature === 'string' && fieldInFeature.charAt(0) === '{') { // data from CSV come as string, if it doesn't start with bracket then it should not be converted to an object and stay as a string, since the structure has been simplified: https://redmine.neogeo.fr/issues/18740
                   try {
                     const jsonStr = fieldInFeature.replace(/['‘’"]\s*label\s*['‘’"]\s*:/g, '"label":')
                       .replace(/:\s*['‘’"](.+?)['‘’"]\s*(?=[,}])/g, ':"$1"');
-                    // Parse la chaîne en JSON
                     fieldInFeature = JSON.parse(jsonStr);
                   } catch (e) {
                     console.error(e);
@@ -638,10 +658,12 @@ export default {
                 const isPreRecordedValue = await this.checkPreRecordedValue(fieldLabel, options[0]);
                 if (!isPreRecordedValue) {
                   this.importError = `Fichier invalide: La valeur "${fieldLabel}" ne fait pas partie des valeurs pré-enregistrées dans le champ "${name}" du signalement "${properties.title}".`;
+                  this.$store.commit('DISCARD_LOADER');
                   return false;
                 }
+
+              // Handle 'multi_choices_list' by checking if each value in the array is among the defined options
               } else if (field_type === 'multi_choices_list') {
-                //*then check if the values in the value array are available options
                 if (typeof fieldInFeature === 'string' && fieldInFeature.charAt(0) === '[') { // data from CSV come as string, if it doesn't start with bracket then there's no need to convert it to an array
                   try {
                     fieldInFeature = JSON.parse(fieldInFeature.replaceAll('\'', '"'));
@@ -652,36 +674,43 @@ export default {
                 }
                 // Check that the value is an array before asserting its validity
                 if (Array.isArray(fieldInFeature)) {
-                  const unvalidValues = fieldInFeature.filter((el) => !options.includes(el));
-                  if (unvalidValues.length > 0) {
-                    const plural = unvalidValues.length > 1;
-                    this.importError = `Fichier invalide: ${plural ? 'Les valeurs' : 'La valeur'} "${unvalidValues.join(', ')}" ${plural ? 'ne sont pas des options valides' : 'n\'est pas une option valide'}
+                  const invalidValues = fieldInFeature.filter((el) => !options.includes(el));
+                  if (invalidValues.length > 0) {
+                    const plural = invalidValues.length > 1;
+                    this.importError = `Fichier invalide: ${plural ? 'Les valeurs' : 'La valeur'} "${invalidValues.join(', ')}" ${plural ? 'ne sont pas des options valides' : 'n\'est pas une option valide'}
                       dans le champ "${name}" du signalement "${properties.title}".`;
+                    this.$store.commit('DISCARD_LOADER');
                     return false;
                   }
                 } else {
                   this.importError = `Fichier invalide: La valeur "${fieldInFeature}" doit être un tableau dans le champ "${name}" du signalement "${properties.title}".`;
+                  this.$store.commit('DISCARD_LOADER');
                   return false;
                 }
-              //* check if custom field value match
+
+              // Validate custom field value type
               } else if (customType !== field_type &&
               // at feature type at creation, in case the value was 0, since it can be either float or integer, by default we've set its type as a float
               // when importing features, to avoid an error with different types, we bypass this check when the incoming feature value is a integer while the feature type says it should be a float
                 !(
+                  // Allow integers where decimals are expected
                   (customType === 'integer' && field_type === 'decimal') ||
-                  // if a number is expected to be formatted as a string, bypass the check, since it won't create an error, all fields from files comes as string anyway
-                  ((customType === 'integer' || customType === 'float') && field_type === 'char') ||
-                  // if expected type is 'text' (multiline string) accept 'char'
+                  // Allow numbers formatted as strings when 'char' or 'text' type is expected
+                  ((customType === 'integer' || customType === 'float') && field_type === 'char' || field_type === 'text') ||
+                  // Allow 'char' values where 'text' (multiline string) is expected
                   (customType === 'char' && field_type === 'text')
                 )
               ) {
                 this.importError = `Fichier invalide : Le type de champ "${field_type}" ne peut pas avoir la valeur "${fieldInFeature}" dans le champ "${name}" du signalement "${properties.title}".`;
+                this.$store.commit('DISCARD_LOADER');
                 return false;
               }
             }
           }
         }
+        count +=1
       }
+      this.$store.commit('DISCARD_LOADER');
       return true;
     },
 

src/views/FeatureType/FeatureTypeEdit.vue

@@ -647,19 +647,64 @@ export default {
       });
     },
 
-    buildCustomForm(properties) {
-      for (const [key, val] of Object.entries(properties)) {
-        //* check that the property is not a keyword from the backend or map style
-        if (!reservedKeywords.includes(key)) {
-          const customForm = {
-            label: { value: key || '' },
-            name: { value: key || '' },
-            position: this.dataKey, // * use dataKey already incremented by addCustomForm
-            field_type: { value: transformProperties(val) }, // * guessed from the type
-            options: { value: [] }, // * not available in export
-          };
-          this.addCustomForm(customForm);
+    /**
+     * Builds custom form fields based on the properties of data entries.
+     *
+     * This function iterates through a subset of data entries (such as rows from a CSV, JSON objects, or GeoJSON features)
+     * to determine the most appropriate type for each field. It tracks confirmed types to avoid redundant checks and 
+     * stops processing a field once its type is definitively determined. If a field is initially detected as a 'char',
+     * it remains as 'char' unless a multiline text ('text') is detected later. The function prioritizes the detection 
+     * of definitive types (like 'text', 'boolean', 'integer') and updates the form with the confirmed types.
+     *
+     * @param {Array} propertiesList - An array of data entries, where each entry is an object representing a set of properties.
+     */
+
+    buildCustomForm(propertiesList) {
+      const confirmedTypes = {};  // Store confirmed types for each field
+      const detectedAsChar = {};  // Track fields initially detected as 'char'
+
+      // Iterate over each row or feature in the subset
+      propertiesList.forEach((properties) => {
+        for (const [key, val] of Object.entries(properties)) {
+          if (!reservedKeywords.includes(key)) {
+
+            // If the type for this field has already been confirmed as something other than 'char', skip it
+            if (confirmedTypes[key] && confirmedTypes[key] !== 'char') {
+              continue;
+            }
+
+            // Determine the type of the current value
+            const detectedType = transformProperties(val);
+
+            if (detectedType === 'text') {
+              // Once 'text' (multiline) is detected, confirm it immediately
+              confirmedTypes[key] = 'text';
+            } else if (!confirmedTypes[key] && detectedType !== 'char') {
+              // If a type is detected that is not 'char' and not yet confirmed, confirm it
+              confirmedTypes[key] = detectedType;
+            } else if (!confirmedTypes[key]) {
+              // If this field hasn't been confirmed yet, initialize it as 'char'
+              confirmedTypes[key] = 'char';
+              detectedAsChar[key] = true;
+            } else if (detectedAsChar[key] && detectedType !== 'char') {
+              // If a field was initially detected as 'char' but now has a different type, update it
+              confirmedTypes[key] = detectedType;
+              delete detectedAsChar[key];  // Remove from 'char' tracking once updated
+            }
+          }
         }
+      });
+
+      // Build custom forms using the confirmed types
+      for (const [key, confirmedType] of Object.entries(confirmedTypes)) {
+        const customForm = {
+          label: { value: key || '' },
+          name: { value: key || '' },
+          position: this.dataKey, // use dataKey already incremented by addCustomForm
+          field_type: { value: confirmedType }, // use the confirmed type
+          options: { value: [] }, // not available in export
+        };
+        this.addCustomForm(customForm);
       }
     },
 
@@ -675,21 +720,27 @@ export default {
 
     importGeoJsonFeatureType() {
       if (this.geojson.features && this.geojson.features.length) {
-        //* in order to get feature_type properties, the first feature is enough
-        const { properties, geometry } = this.geojson.features[0];
-        this.form.title.value = properties.feature_type;
+        const { geometry } = this.geojson.features[0];
         this.form.geom_type.value = geometry.type.toLowerCase();
-        this.updateStore(); //* register title & geom_type in store
-        this.buildCustomForm(properties);
+        this.updateStore(); // register geom_type in store
+
+        // Use a subset of the first N features to build the form
+        const subsetFeatures = this.geojson.features.slice(0, 200); // Adjust '200' based on performance needs
+        const propertiesList = subsetFeatures.map(feature => feature.properties);
+        this.buildCustomForm(propertiesList);
       }
       this.setTitleFromFile();
     },
 
     importCSVFeatureType() {
       if (this.csv.length) {
-        this.updateStore(); //* register title & geom_type in store
-        this.buildCustomForm(this.csv[0]);
-        // if the first csv element doesn't contain geom data, set the feature type as non geom
+        this.updateStore(); // register title in store
+
+        // Use a subset of the first N rows to build the form
+        const subsetCSV = this.csv.slice(0, 200); // Adjust '200' based on performance needs
+        this.buildCustomForm(subsetCSV);
+
+        // Check for geom data
         if (!('lat' in this.csv[0]) || !('lon' in this.csv[0])) {
           this.form.geom_type.value = 'none';
         }
@@ -699,13 +750,16 @@ export default {
 
     importJsonFeatureType() {
       if (this.json.length) {
-        // JSON are non geom features
-        this.form.geom_type.value = 'none';
-        this.updateStore(); //* register title & geom_type in store
-        this.buildCustomForm(this.json[0]);
+        this.form.geom_type.value = 'none'; // JSON are non-geom features
+        this.updateStore(); // register title in store
+
+        // Use a subset of the first N objects to build the form
+        const subsetJson = this.json.slice(0, 200); // Adjust '200' based on performance needs
+        this.buildCustomForm(subsetJson);
       }
       this.setTitleFromFile();
     },
+
   },
 };
 </script>

src/views/Project/ProjectDetail.vue

@@ -14,7 +14,7 @@
         <div class="row">
           <div class="eight wide column">
             <ProjectFeatureTypes
-              :loading="featureTypesLoading || projectInfoLoading"
+              :loading="featureTypesLoading"
               :project="project"
               @delete="toggleDeleteFeatureTypeModal"
               @update="updateAfterImport"