Skip to content
Snippets Groups Projects
Commit 088f3353 authored by Timothee P's avatar Timothee P :sunflower:
Browse files

fix: improve type detection & display loader while checking values

parent 25fd4ee7
No related branches found
No related tags found
1 merge request!844REDMINE_ISSUE-23123 | Import impossible d'un signalement
......@@ -66,31 +66,45 @@ export function allowedStatus2change(user, isModerate, userStatus, isOwnFeature,
return [];
}
/**
* Determines the type of a property based on its value.
*
* This function inspects a given property and returns a string indicating its type,
* such as 'boolean', 'integer', 'decimal', 'date', 'text', or 'char'.
* It uses various checks to determine the appropriate type for different value formats.
*
* @param {any} prop - The property value to be evaluated.
* @returns {string} The determined type of the property ('boolean', 'integer', 'decimal', 'date', 'text', or 'char').
*/
export function transformProperties(prop) {
const type = typeof prop;
const date = new Date(prop);
const regInteger = /^-*?\d+$/;
const regFloat = /^-*?\d*?\.\d+$/;
const regText = /[\r\n]/;
if (type === 'boolean' || (type === 'string' && (prop.toLowerCase() === 'true' || prop.toLowerCase() === 'False'))) {
const regInteger = /^-?\d+$/; // Regular expression to match integer numbers
const regFloat = /^-?\d*\.\d+$/; // Regular expression to match decimal numbers
const regText = /[\r\n]/; // Regular expression to detect multiline text (newlines)
const regDate = /^\d{4}-\d{2}-\d{2}$|^\d{2}\/\d{2}\/\d{4}$/; // Regular expression to match common date formats
// Check if the property is a boolean or a string that represents a boolean
if (type === 'boolean' || (type === 'string' && (prop.toLowerCase() === 'true' || prop.toLowerCase() === 'false'))) {
return 'boolean';
} else if ((regInteger.test(prop) || Number.isSafeInteger(prop)) && prop !== 0) { // in case the value is 0, since it can be either float or integer, do not set to integer https://redmine.neogeo.fr/issues/16934
} else if (regInteger.test(prop) || (type === 'number' && Number.isSafeInteger(prop))) {
// Check if the property is an integer or a string that represents an integer
return 'integer';
} else if (
type === 'string' &&
['/', ':', '-'].some((el) => prop.includes(el)) && // check for chars found in datestring
date instanceof Date &&
!isNaN(date.valueOf())
) {
} else if (type === 'string' && regDate.test(prop.trim())) {
// More specific check for date strings using regular expressions
return 'date';
} else if (regFloat.test(prop) || type === 'number' && !isNaN(parseFloat(prop)) || prop === 0) { // in case the value is 0, since it can be either float or integer, by default set as a float
} else if (regFloat.test(prop) || (type === 'number' && !Number.isSafeInteger(prop))) {
// Check if the property is a decimal number or a string that represents a decimal
return 'decimal';
} else if (regText.test(prop)) {
} else if (regText.test(prop) || (type === 'string' && prop.length > 255)) {
// Check if the property contains newline characters or is a long text
return 'text';
}
return 'char'; //* string by default, most accepted type in database
// Default case for all other types: assume it is a short text or character field
return 'char';
}
export function objIsEmpty(obj) {
for(const prop in obj) {
if(Object.hasOwn(obj, prop)) {
......
......@@ -593,8 +593,21 @@ export default {
return this.selectedPrerecordedListValues[listName].some((el) => el.label === fieldLabel);
},
/**
* Validates the imported data against the pre-determined field types.
*
* This function iterates over all imported features and checks if each property's value matches
* the expected type specified in the feature type schema. It accommodates specific type conversions,
* such as allowing numerical strings for 'char' or 'text' fields and converting string representations
* of booleans and lists as necessary.
*
* @param {Array} features - The array of imported features to validate.
* @returns {boolean} Returns true if all features pass the validation; otherwise, false with an error message.
*/
async isValidTypes(features) {
this.importError = '';
// Extract relevant field type information from the feature type schema
const fields = this.feature_type.customfield_set.map((el) => {
return {
name: el.name,
......@@ -602,32 +615,39 @@ export default {
options: el.options,
};
});
let count = 1;
for (const feature of features) {
this.$store.commit('DISPLAY_LOADER', `Vérification du signalement ${count} sur ${features.length}`);
for (const { name, field_type, options } of fields) {
const properties = feature.properties || feature;
if (name in properties) {
let fieldInFeature = properties[name];
if (field_type === 'boolean') { // in csv booleans are capitalized string that need to be translated in javascript syntax
// Convert boolean strings from CSV to actual booleans
if (field_type === 'boolean') {
fieldInFeature = fieldInFeature === 'True' ? true : (fieldInFeature === 'False' ? false : fieldInFeature);
}
const customType = transformProperties(fieldInFeature);
//* if custom field value is not defined or not null or in case of prerecorded list with defined value => then check validity of field type
// Validate field only if it has a non-null, non-empty, defined value
if (fieldInFeature !== null && fieldInFeature !== '' && fieldInFeature !== undefined) {
//* if field type is list, it's not possible to guess from value type
// Handle 'list' type by checking if value is among the defined options
if (field_type === 'list') {
//*then check if the value is an available option
if (!options.includes(fieldInFeature)) {
this.importError = `Fichier invalide: La valeur "${fieldInFeature}" n'est pas une option valide dans le champ "${name}" du signalement "${properties.title}".`;
this.$store.commit('DISCARD_LOADER');
return false;
}
// Handle 'pre_recorded_list' by checking if the value matches pre-recorded options
} else if (field_type === 'pre_recorded_list') {
if (typeof fieldInFeature === 'string' && fieldInFeature.charAt(0) === '{') { // data from CSV come as string, if it doesn't start with bracket then it should not be converted to an object and stay as a string, since the structure has been simplified: https://redmine.neogeo.fr/issues/18740
try {
const jsonStr = fieldInFeature.replace(/['‘’"]\s*label\s*['‘’"]\s*:/g, '"label":')
.replace(/:\s*['‘’"](.+?)['‘’"]\s*(?=[,}])/g, ':"$1"');
// Parse la chaîne en JSON
fieldInFeature = JSON.parse(jsonStr);
} catch (e) {
console.error(e);
......@@ -638,10 +658,12 @@ export default {
const isPreRecordedValue = await this.checkPreRecordedValue(fieldLabel, options[0]);
if (!isPreRecordedValue) {
this.importError = `Fichier invalide: La valeur "${fieldLabel}" ne fait pas partie des valeurs pré-enregistrées dans le champ "${name}" du signalement "${properties.title}".`;
this.$store.commit('DISCARD_LOADER');
return false;
}
// Handle 'multi_choices_list' by checking if each value in the array is among the defined options
} else if (field_type === 'multi_choices_list') {
//*then check if the values in the value array are available options
if (typeof fieldInFeature === 'string' && fieldInFeature.charAt(0) === '[') { // data from CSV come as string, if it doesn't start with bracket then there's no need to convert it to an array
try {
fieldInFeature = JSON.parse(fieldInFeature.replaceAll('\'', '"'));
......@@ -652,36 +674,43 @@ export default {
}
// Check that the value is an array before asserting its validity
if (Array.isArray(fieldInFeature)) {
const unvalidValues = fieldInFeature.filter((el) => !options.includes(el));
if (unvalidValues.length > 0) {
const plural = unvalidValues.length > 1;
this.importError = `Fichier invalide: ${plural ? 'Les valeurs' : 'La valeur'} "${unvalidValues.join(', ')}" ${plural ? 'ne sont pas des options valides' : 'n\'est pas une option valide'}
const invalidValues = fieldInFeature.filter((el) => !options.includes(el));
if (invalidValues.length > 0) {
const plural = invalidValues.length > 1;
this.importError = `Fichier invalide: ${plural ? 'Les valeurs' : 'La valeur'} "${invalidValues.join(', ')}" ${plural ? 'ne sont pas des options valides' : 'n\'est pas une option valide'}
dans le champ "${name}" du signalement "${properties.title}".`;
this.$store.commit('DISCARD_LOADER');
return false;
}
} else {
this.importError = `Fichier invalide: La valeur "${fieldInFeature}" doit être un tableau dans le champ "${name}" du signalement "${properties.title}".`;
this.$store.commit('DISCARD_LOADER');
return false;
}
//* check if custom field value match
// Validate custom field value type
} else if (customType !== field_type &&
// at feature type at creation, in case the value was 0, since it can be either float or integer, by default we've set its type as a float
// when importing features, to avoid an error with different types, we bypass this check when the incoming feature value is a integer while the feature type says it should be a float
!(
// Allow integers where decimals are expected
(customType === 'integer' && field_type === 'decimal') ||
// if a number is expected to be formatted as a string, bypass the check, since it won't create an error, all fields from files comes as string anyway
((customType === 'integer' || customType === 'float') && field_type === 'char') ||
// if expected type is 'text' (multiline string) accept 'char'
// Allow numbers formatted as strings when 'char' or 'text' type is expected
((customType === 'integer' || customType === 'float') && field_type === 'char' || field_type === 'text') ||
// Allow 'char' values where 'text' (multiline string) is expected
(customType === 'char' && field_type === 'text')
)
) {
this.importError = `Fichier invalide : Le type de champ "${field_type}" ne peut pas avoir la valeur "${fieldInFeature}" dans le champ "${name}" du signalement "${properties.title}".`;
this.$store.commit('DISCARD_LOADER');
return false;
}
}
}
}
count +=1
}
this.$store.commit('DISCARD_LOADER');
return true;
},
......
......@@ -647,19 +647,64 @@ export default {
});
},
buildCustomForm(properties) {
for (const [key, val] of Object.entries(properties)) {
//* check that the property is not a keyword from the backend or map style
if (!reservedKeywords.includes(key)) {
const customForm = {
label: { value: key || '' },
name: { value: key || '' },
position: this.dataKey, // * use dataKey already incremented by addCustomForm
field_type: { value: transformProperties(val) }, // * guessed from the type
options: { value: [] }, // * not available in export
};
this.addCustomForm(customForm);
/**
* Builds custom form fields based on the properties of data entries.
*
* This function iterates through a subset of data entries (such as rows from a CSV, JSON objects, or GeoJSON features)
* to determine the most appropriate type for each field. It tracks confirmed types to avoid redundant checks and
* stops processing a field once its type is definitively determined. If a field is initially detected as a 'char',
* it remains as 'char' unless a multiline text ('text') is detected later. The function prioritizes the detection
* of definitive types (like 'text', 'boolean', 'integer') and updates the form with the confirmed types.
*
* @param {Array} propertiesList - An array of data entries, where each entry is an object representing a set of properties.
*/
buildCustomForm(propertiesList) {
const confirmedTypes = {}; // Store confirmed types for each field
const detectedAsChar = {}; // Track fields initially detected as 'char'
// Iterate over each row or feature in the subset
propertiesList.forEach((properties) => {
for (const [key, val] of Object.entries(properties)) {
if (!reservedKeywords.includes(key)) {
// If the type for this field has already been confirmed as something other than 'char', skip it
if (confirmedTypes[key] && confirmedTypes[key] !== 'char') {
continue;
}
// Determine the type of the current value
const detectedType = transformProperties(val);
if (detectedType === 'text') {
// Once 'text' (multiline) is detected, confirm it immediately
confirmedTypes[key] = 'text';
} else if (!confirmedTypes[key] && detectedType !== 'char') {
// If a type is detected that is not 'char' and not yet confirmed, confirm it
confirmedTypes[key] = detectedType;
} else if (!confirmedTypes[key]) {
// If this field hasn't been confirmed yet, initialize it as 'char'
confirmedTypes[key] = 'char';
detectedAsChar[key] = true;
} else if (detectedAsChar[key] && detectedType !== 'char') {
// If a field was initially detected as 'char' but now has a different type, update it
confirmedTypes[key] = detectedType;
delete detectedAsChar[key]; // Remove from 'char' tracking once updated
}
}
}
});
// Build custom forms using the confirmed types
for (const [key, confirmedType] of Object.entries(confirmedTypes)) {
const customForm = {
label: { value: key || '' },
name: { value: key || '' },
position: this.dataKey, // use dataKey already incremented by addCustomForm
field_type: { value: confirmedType }, // use the confirmed type
options: { value: [] }, // not available in export
};
this.addCustomForm(customForm);
}
},
......@@ -675,21 +720,27 @@ export default {
importGeoJsonFeatureType() {
if (this.geojson.features && this.geojson.features.length) {
//* in order to get feature_type properties, the first feature is enough
const { properties, geometry } = this.geojson.features[0];
this.form.title.value = properties.feature_type;
const { geometry } = this.geojson.features[0];
this.form.geom_type.value = geometry.type.toLowerCase();
this.updateStore(); //* register title & geom_type in store
this.buildCustomForm(properties);
this.updateStore(); // register geom_type in store
// Use a subset of the first N features to build the form
const subsetFeatures = this.geojson.features.slice(0, 200); // Adjust '200' based on performance needs
const propertiesList = subsetFeatures.map(feature => feature.properties);
this.buildCustomForm(propertiesList);
}
this.setTitleFromFile();
},
importCSVFeatureType() {
if (this.csv.length) {
this.updateStore(); //* register title & geom_type in store
this.buildCustomForm(this.csv[0]);
// if the first csv element doesn't contain geom data, set the feature type as non geom
this.updateStore(); // register title in store
// Use a subset of the first N rows to build the form
const subsetCSV = this.csv.slice(0, 200); // Adjust '200' based on performance needs
this.buildCustomForm(subsetCSV);
// Check for geom data
if (!('lat' in this.csv[0]) || !('lon' in this.csv[0])) {
this.form.geom_type.value = 'none';
}
......@@ -699,13 +750,16 @@ export default {
importJsonFeatureType() {
if (this.json.length) {
// JSON are non geom features
this.form.geom_type.value = 'none';
this.updateStore(); //* register title & geom_type in store
this.buildCustomForm(this.json[0]);
this.form.geom_type.value = 'none'; // JSON are non-geom features
this.updateStore(); // register title in store
// Use a subset of the first N objects to build the form
const subsetJson = this.json.slice(0, 200); // Adjust '200' based on performance needs
this.buildCustomForm(subsetJson);
}
this.setTitleFromFile();
},
},
};
</script>
......
......@@ -14,7 +14,7 @@
<div class="row">
<div class="eight wide column">
<ProjectFeatureTypes
:loading="featureTypesLoading || projectInfoLoading"
:loading="featureTypesLoading"
:project="project"
@delete="toggleDeleteFeatureTypeModal"
@update="updateAfterImport"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment