Skip to content

delete fileDate from new surveyrUnitModels and replace them with rawRecordDate #482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Hajarel-moukh wants to merge 1 commit into
base: main
Choose a base branch
from
Open

delete fileDate from new surveyrUnitModels and replace them with rawRecordDate #482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,7 @@ private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit s
.state(dataState)
.mode(mode)
.recordDate(Instant.now())
.fileDate(su.getFileDate())
.rawRecordDate(su.getFileDate())
.rawRecordDate(su.getRawRecordDate())
.build();

return getCollectedDataFromSurveyUnit(su, surveyUnitModel, variablesMap, dataState);
Expand Down
5 changes: 2 additions & 3 deletions src/main/java/fr/insee/genesis/controller/sources/xml/LunaticXmlDataParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ private Document readXmlFile(Path filePath) throws IOException, SAXException, Ge
return document;
}

private LocalDateTime getFileDate(Path filePath) throws IOException {
private LocalDateTime getRawRecordDate(Path filePath) throws IOException {
BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class);
return LocalDateTime.ofInstant(attr.lastModifiedTime().toInstant(), ZoneId.of("Europe/Paris"));
}
Expand All @@ -68,9 +68,8 @@ public LunaticXmlCampaign parseDataFile(Path filePath) throws GenesisException,
if (surveyUnit.getNodeType() == Node.ELEMENT_NODE) {
Element surveyUnitElement = (Element) surveyUnit;
LunaticXmlSurveyUnit lunaticXmlSurveyUnit = new LunaticXmlSurveyUnit();
LocalDateTime rawRecordDate = getFileDate(filePath);
LocalDateTime rawRecordDate = getRawRecordDate(filePath);
lunaticXmlSurveyUnit.setRawRecordDate(rawRecordDate);
lunaticXmlSurveyUnit.setFileDate(rawRecordDate);
lunaticXmlSurveyUnit.setId(surveyUnitElement.getElementsByTagName("Id").item(0).getFirstChild().getNodeValue());
lunaticXmlSurveyUnit.setQuestionnaireModelId(surveyUnitElement.getElementsByTagName("QuestionnaireModelId").item(0).getFirstChild().getNodeValue());
Node data = surveyUnitElement.getElementsByTagName("Data").item(0);
Expand Down
5 changes: 2 additions & 3 deletions src/main/java/fr/insee/genesis/controller/sources/xml/LunaticXmlDataSequentialParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public class LunaticXmlDataSequentialParser{


public LunaticXmlDataSequentialParser(final Path filePath, final InputStream stream) throws IOException, XMLStreamException {
this.rawRecordDate = getFileDate(filePath);
this.rawRecordDate = getRawRecordDate(filePath);

XMLInputFactory factory = XMLInputFactory.newInstance();
factory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
Expand Down Expand Up @@ -90,7 +90,6 @@ public LunaticXmlSurveyUnit readNextSurveyUnit() throws XMLStreamException {
*/
private LunaticXmlSurveyUnit parseSurveyUnit(final XMLEventReader reader) throws XMLStreamException {
LunaticXmlSurveyUnit xmlSurveyUnit = new LunaticXmlSurveyUnit();
xmlSurveyUnit.setFileDate(this.rawRecordDate);
xmlSurveyUnit.setRawRecordDate(this.rawRecordDate);

LunaticXmlData data = new LunaticXmlData();
Expand Down Expand Up @@ -276,7 +275,7 @@ private List<LunaticXmlOtherData> readCalculatedOrExternal(XMLEventReader reader
return lunaticXmlOtherDataList;
}

private LocalDateTime getFileDate(Path filePath) throws IOException {
private LocalDateTime getRawRecordDate(Path filePath) throws IOException {
BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class);
return LocalDateTime.ofInstant(attr.lastModifiedTime().toInstant(), ZoneId.of("Europe/Paris"));
}
Expand Down
1 change: 0 additions & 1 deletion src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ public List<SurveyUnitModel> convertRawDataAndCollectEmptyModels(
.validationDate(payloadParser.getValidationDate(rawData))
.isCapturedIndirectly(payloadParser.getIsCapturedIndirectly(rawData))
.state(dataState)
.fileDate(rawData.recordDate())
.rawRecordDate(rawData.recordDate())
.recordDate(Instant.now())
.collectedVariables(new ArrayList<>())
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ private SurveyUnitModel buildSurveyUnitModel(RawResponseModel rawResponseModel,
.validationDate(rawResponsePayloadParser.getValidationDate(rawResponseModel))
.isCapturedIndirectly(rawResponsePayloadParser.getIsCapturedIndirectly(rawResponseModel))
.state(dataState)
.fileDate(rawResponseModel.recordDate())
.rawRecordDate(rawResponseModel.recordDate())
.recordDate(Instant.now())
.collectedVariables(new ArrayList<>())
.externalVariables(new ArrayList<>())
Expand Down
1 change: 0 additions & 1 deletion src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,6 @@ private static SurveyUnitModel createFormattedSurveyUnitModel(
.state(DataState.FORMATTED)
.mode(sampleSurveyUnitModel.getMode())
.recordDate(Instant.now().plusSeconds(1)) // Add 1 second to avoid same recordDate as COLLECTED
.fileDate(sampleSurveyUnitModel.getFileDate())
.rawRecordDate(sampleSurveyUnitModel.getRawRecordDate())
.collectedVariables(new ArrayList<>())
.externalVariables(new ArrayList<>())
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ public interface SurveyUnitDocumentMapper {
@Mapping(target = "mode", source = "mode", qualifiedByName = "mapMode")
SurveyUnitModel documentToModel(SurveyUnitDocument surveyUnit);

@Mapping(target = "fileDate", ignore = true)
SurveyUnitDocument modelToDocument(SurveyUnitModel surveyUnitModel);

@Mapping(target = "mode", source = "mode", qualifiedByName = "mapMode")
Expand All @@ -39,6 +40,10 @@ default void handleDeprecatedFields(SurveyUnitDocument doc,
model.setCollectionInstrumentId(doc.getQuestionnaireId());
}

if (model.getRawRecordDate() == null) {
model.setRawRecordDate(doc.getFileDate());
}

}

@Named("mapMode")
Expand Down
2 changes: 0 additions & 2 deletions src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,14 +255,12 @@ void convert_shouldMapRawRecordDate() {
// GIVEN
LocalDateTime rawRecordDate = LocalDateTime.of(2024, 1, 15, 10, 0);
LunaticXmlSurveyUnit su = buildSurveyUnit(List.of(collectedDataWithValue("VAR1", "val")));
su.setFileDate(rawRecordDate);
su.setRawRecordDate(rawRecordDate);

// WHEN
SurveyUnitModel collected = getCollected(LunaticXmlAdapter.convert(su, VARIABLES_MAP, MODE));

// THEN
assertThat(collected.getFileDate()).isEqualTo(rawRecordDate);
assertThat(collected.getRawRecordDate()).isEqualTo(rawRecordDate);
}
}
Expand Down
3 changes: 0 additions & 3 deletions src/test/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModelTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ void noArgsConstructor_shouldCreateEmptyModel() {
assertThat(model.getQuestionnaireState()).isNull();
assertThat(model.getValidationDate()).isNull();
assertThat(model.getRecordDate()).isNull();
assertThat(model.getFileDate()).isNull();
assertThat(model.getRawRecordDate()).isNull();
assertThat(model.getCollectedVariables()).isNull();
assertThat(model.getExternalVariables()).isNull();
Expand Down Expand Up @@ -112,7 +111,6 @@ void allArgsConstructor_shouldSetAllFields() {
assertThat(model.getQuestionnaireState()).isEqualTo(RawResponseDto.QuestionnaireStateEnum.FINISHED);
assertThat(model.getValidationDate()).isEqualTo(now);
assertThat(model.getRecordDate()).isEqualTo(now.toInstant(ZoneOffset.UTC));
assertThat(model.getFileDate()).isEqualTo(now);
assertThat(model.getRawRecordDate()).isEqualTo(now);
assertThat(model.getCollectedVariables()).isEqualTo(collected);
assertThat(model.getExternalVariables()).isEqualTo(external);
Expand Down Expand Up @@ -164,7 +162,6 @@ void builder_shouldSetAllFields() {
assertThat(model.getQuestionnaireState()).isEqualTo(RawResponseDto.QuestionnaireStateEnum.FINISHED);
assertThat(model.getValidationDate()).isEqualTo(now);
assertThat(model.getRecordDate()).isEqualTo(now.toInstant(ZoneOffset.UTC));
assertThat(model.getFileDate()).isEqualTo(now);
assertThat(model.getRawRecordDate()).isEqualTo(now);
assertThat(model.getCollectedVariables()).isEqualTo(collected);
assertThat(model.getExternalVariables()).isEqualTo(external);
Expand Down
8 changes: 0 additions & 8 deletions src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ static void init(){
surveyUnitDocumentStatic.setInterrogationId(INTERROGATION_ID);
surveyUnitDocumentStatic.setState("COLLECTED");
LocalDateTime rawRecordDate = LocalDateTime.of(2023, 1, 1, 0, 0, 0);
surveyUnitDocumentStatic.setFileDate(rawRecordDate);
surveyUnitDocumentStatic.setRawRecordDate(rawRecordDate);

List<VariableDocument> documentExternalVariableList = new ArrayList<>();
Expand Down Expand Up @@ -90,7 +89,6 @@ static void init(){
deprecatedSurveyUnitDocumentStatic.setIdUE(ID_UE);
deprecatedSurveyUnitDocumentStatic.setInterrogationId(INTERROGATION_ID);
deprecatedSurveyUnitDocumentStatic.setState("COLLECTED");
deprecatedSurveyUnitDocumentStatic.setFileDate(rawRecordDate);
deprecatedSurveyUnitDocumentStatic.setRawRecordDate(rawRecordDate);

documentExternalVariableList = new ArrayList<>();
Expand Down Expand Up @@ -158,7 +156,6 @@ void shouldReturnModelFromDocument(){
Assertions.assertThat(surveyUnit.getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
Assertions.assertThat(surveyUnit.getUsualSurveyUnitId()).isEqualTo(USUAL_SURVEY_UNIT_ID);
Assertions.assertThat(surveyUnit.getState()).isEqualTo(DataState.COLLECTED);
Assertions.assertThat(surveyUnit.getFileDate()).isEqualTo(rawRecordDate);
Assertions.assertThat(surveyUnit.getRawRecordDate()).isEqualTo(rawRecordDate);

Assertions.assertThat(surveyUnit.getExternalVariables()).filteredOn(externalVariableModel ->
Expand All @@ -184,7 +181,6 @@ void shouldReturnModelFromDeprecatedDocument(){
Assertions.assertThat(surveyUnit.getCollectionInstrumentId()).isEqualTo(QUESTIONNAIRE_ID);
Assertions.assertThat(surveyUnit.getUsualSurveyUnitId()).isEqualTo(ID_UE);
Assertions.assertThat(surveyUnit.getState()).isEqualTo(DataState.COLLECTED);
Assertions.assertThat(surveyUnit.getFileDate()).isEqualTo(rawRecordDate);
Assertions.assertThat(surveyUnit.getRawRecordDate()).isEqualTo(rawRecordDate);

Assertions.assertThat(surveyUnit.getExternalVariables()).filteredOn(externalVariableModel ->
Expand All @@ -209,7 +205,6 @@ void shouldReturnDocumentFromModel(){
Assertions.assertThat(surveyUnitDocument.getInterrogationId()).isEqualTo(INTERROGATION_ID);
Assertions.assertThat(surveyUnitDocument.getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
Assertions.assertThat(surveyUnitDocument.getState()).isEqualTo("COLLECTED");
Assertions.assertThat(surveyUnitDocument.getFileDate()).isEqualTo(rawRecordDate);
Assertions.assertThat(surveyUnitDocument.getRawRecordDate()).isEqualTo(rawRecordDate);

Assertions.assertThat(surveyUnitDocument.getExternalVariables()).filteredOn(externalVariableDocument ->
Expand Down Expand Up @@ -239,7 +234,6 @@ void shouldReturnModelListFromDocumentList(){
Assertions.assertThat(surveyUnitList.getFirst().getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
Assertions.assertThat(surveyUnitList.getFirst().getUsualSurveyUnitId()).isEqualTo(USUAL_SURVEY_UNIT_ID);
Assertions.assertThat(surveyUnitList.getFirst().getState()).isEqualTo(DataState.COLLECTED);
Assertions.assertThat(surveyUnitList.getFirst().getFileDate()).isEqualTo(rawRecordDate);
Assertions.assertThat(surveyUnitList.getFirst().getRawRecordDate()).isEqualTo(rawRecordDate);

Assertions.assertThat(surveyUnitList.getFirst().getExternalVariables()).filteredOn(externalVariableModel ->
Expand Down Expand Up @@ -267,7 +261,6 @@ void shouldReturnModelListFromDeprecatedDocumentList(){
Assertions.assertThat(surveyUnitList.getFirst().getCollectionInstrumentId()).isEqualTo(QUESTIONNAIRE_ID);
Assertions.assertThat(surveyUnitList.getFirst().getUsualSurveyUnitId()).isEqualTo(ID_UE);
Assertions.assertThat(surveyUnitList.getFirst().getState()).isEqualTo(DataState.COLLECTED);
Assertions.assertThat(surveyUnitList.getFirst().getFileDate()).isEqualTo(rawRecordDate);
Assertions.assertThat(surveyUnitList.getFirst().getRawRecordDate()).isEqualTo(rawRecordDate);

Assertions.assertThat(surveyUnitList.getFirst().getExternalVariables()).filteredOn(externalVariableModel ->
Expand All @@ -293,7 +286,6 @@ void shouldReturnDocumentListFromModelList(){
Assertions.assertThat(surveyUnitDocumentList.getFirst().getInterrogationId()).isEqualTo(INTERROGATION_ID);
Assertions.assertThat(surveyUnitDocumentList.getFirst().getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
Assertions.assertThat(surveyUnitDocumentList.getFirst().getState()).isEqualTo("COLLECTED");
Assertions.assertThat(surveyUnitDocumentList.getFirst().getFileDate()).isEqualTo(rawRecordDate);
Assertions.assertThat(surveyUnitDocumentList.getFirst().getRawRecordDate()).isEqualTo(rawRecordDate);

Assertions.assertThat(surveyUnitDocumentList.getFirst().getExternalVariables()).filteredOn(externalVariableDocument ->
Expand Down
Loading