From 62a89d799969b1de8ac208543e89a7da85345a91 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 17 Feb 2024 11:13:49 -0800 Subject: [PATCH] make sure we correctly extract text narration field from FHIR resources. Convert to Markdown. --- backend/pkg/models/database/fhir_account.go | 8 +++- .../pkg/models/database/fhir_adverse_event.go | 8 +++- .../database/fhir_allergy_intolerance.go | 8 +++- .../pkg/models/database/fhir_appointment.go | 8 +++- backend/pkg/models/database/fhir_binary.go | 8 +++- backend/pkg/models/database/fhir_care_plan.go | 8 +++- backend/pkg/models/database/fhir_care_team.go | 8 +++- backend/pkg/models/database/fhir_claim.go | 8 +++- .../models/database/fhir_claim_response.go | 8 +++- .../pkg/models/database/fhir_composition.go | 8 +++- backend/pkg/models/database/fhir_condition.go | 8 +++- backend/pkg/models/database/fhir_consent.go | 8 +++- backend/pkg/models/database/fhir_coverage.go | 8 +++- .../fhir_coverage_eligibility_request.go | 8 +++- .../fhir_coverage_eligibility_response.go | 8 +++- backend/pkg/models/database/fhir_device.go | 8 +++- .../models/database/fhir_device_request.go | 8 +++- .../models/database/fhir_diagnostic_report.go | 8 +++- .../models/database/fhir_document_manifest.go | 8 +++- .../database/fhir_document_reference.go | 8 +++- backend/pkg/models/database/fhir_encounter.go | 8 +++- backend/pkg/models/database/fhir_endpoint.go | 8 +++- .../database/fhir_enrollment_request.go | 8 +++- .../database/fhir_enrollment_response.go | 8 +++- .../database/fhir_explanation_of_benefit.go | 8 +++- .../database/fhir_family_member_history.go | 8 +++- backend/pkg/models/database/fhir_goal.go | 8 +++- .../pkg/models/database/fhir_imaging_study.go | 8 +++- .../pkg/models/database/fhir_immunization.go | 8 +++- .../models/database/fhir_insurance_plan.go | 8 +++- backend/pkg/models/database/fhir_location.go | 8 +++- backend/pkg/models/database/fhir_media.go | 8 +++- .../pkg/models/database/fhir_medication.go | 8 +++- .../fhir_medication_administration.go | 8 +++- .../database/fhir_medication_dispense.go | 8 +++- .../database/fhir_medication_request.go | 8 +++- .../database/fhir_medication_statement.go | 8 +++- .../models/database/fhir_nutrition_order.go | 8 +++- .../pkg/models/database/fhir_observation.go | 8 +++- .../pkg/models/database/fhir_organization.go | 8 +++- .../database/fhir_organization_affiliation.go | 8 +++- backend/pkg/models/database/fhir_patient.go | 8 +++- backend/pkg/models/database/fhir_person.go | 8 +++- .../pkg/models/database/fhir_practitioner.go | 8 +++- .../models/database/fhir_practitioner_role.go | 8 +++- backend/pkg/models/database/fhir_procedure.go | 8 +++- .../pkg/models/database/fhir_provenance.go | 8 +++- .../pkg/models/database/fhir_questionnaire.go | 8 +++- .../database/fhir_questionnaire_response.go | 8 +++- .../models/database/fhir_related_person.go | 8 +++- backend/pkg/models/database/fhir_schedule.go | 8 +++- .../models/database/fhir_service_request.go | 8 +++- backend/pkg/models/database/fhir_slot.go | 8 +++- backend/pkg/models/database/fhir_specimen.go | 8 +++- .../database/fhir_vision_prescription.go | 8 +++- backend/pkg/models/database/generate.go | 10 ++++- go.mod | 13 ++++--- go.sum | 37 ++++++++++++++----- 58 files changed, 429 insertions(+), 71 deletions(-) diff --git a/backend/pkg/models/database/fhir_account.go b/backend/pkg/models/database/fhir_account.go index 1367e360..a3d0df03 100644 --- a/backend/pkg/models/database/fhir_account.go +++ b/backend/pkg/models/database/fhir_account.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -188,9 +189,14 @@ func (s *FhirAccount) PopulateAndExtractSearchParameters(resourceRaw json.RawMes s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_adverse_event.go b/backend/pkg/models/database/fhir_adverse_event.go index c07d8e98..6930c097 100644 --- a/backend/pkg/models/database/fhir_adverse_event.go +++ b/backend/pkg/models/database/fhir_adverse_event.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -242,9 +243,14 @@ func (s *FhirAdverseEvent) PopulateAndExtractSearchParameters(resourceRaw json.R s.Substance = []byte(substanceResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_allergy_intolerance.go b/backend/pkg/models/database/fhir_allergy_intolerance.go index 6e1f922d..41f43f01 100644 --- a/backend/pkg/models/database/fhir_allergy_intolerance.go +++ b/backend/pkg/models/database/fhir_allergy_intolerance.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -341,9 +342,14 @@ func (s *FhirAllergyIntolerance) PopulateAndExtractSearchParameters(resourceRaw s.Severity = []byte(severityResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting VerificationStatus verificationStatusResult, err := vm.RunString("extractTokenSearchParameters(fhirResource, 'AllergyIntolerance.verificationStatus')") diff --git a/backend/pkg/models/database/fhir_appointment.go b/backend/pkg/models/database/fhir_appointment.go index 43712266..8f89f93f 100644 --- a/backend/pkg/models/database/fhir_appointment.go +++ b/backend/pkg/models/database/fhir_appointment.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -278,9 +279,14 @@ func (s *FhirAppointment) PopulateAndExtractSearchParameters(resourceRaw json.Ra s.SupportingInfo = []byte(supportingInfoResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_binary.go b/backend/pkg/models/database/fhir_binary.go index ca8bb93b..b2c45b7d 100644 --- a/backend/pkg/models/database/fhir_binary.go +++ b/backend/pkg/models/database/fhir_binary.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -126,9 +127,14 @@ func (s *FhirBinary) PopulateAndExtractSearchParameters(resourceRaw json.RawMess s.MetaVersionId = metaVersionIdResult.String() } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_care_plan.go b/backend/pkg/models/database/fhir_care_plan.go index 147ae955..23ccf484 100644 --- a/backend/pkg/models/database/fhir_care_plan.go +++ b/backend/pkg/models/database/fhir_care_plan.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -366,9 +367,14 @@ func (s *FhirCarePlan) PopulateAndExtractSearchParameters(resourceRaw json.RawMe s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_care_team.go b/backend/pkg/models/database/fhir_care_team.go index 621f7ab0..21940f52 100644 --- a/backend/pkg/models/database/fhir_care_team.go +++ b/backend/pkg/models/database/fhir_care_team.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -250,9 +251,14 @@ func (s *FhirCareTeam) PopulateAndExtractSearchParameters(resourceRaw json.RawMe s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_claim.go b/backend/pkg/models/database/fhir_claim.go index 5db75b89..3d5f38e8 100644 --- a/backend/pkg/models/database/fhir_claim.go +++ b/backend/pkg/models/database/fhir_claim.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -273,9 +274,14 @@ func (s *FhirClaim) PopulateAndExtractSearchParameters(resourceRaw json.RawMessa s.SubdetailUdi = []byte(subdetailUdiResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting Use useResult, err := vm.RunString("extractTokenSearchParameters(fhirResource, 'Claim.use')") diff --git a/backend/pkg/models/database/fhir_claim_response.go b/backend/pkg/models/database/fhir_claim_response.go index ae5cfae3..3d396c46 100644 --- a/backend/pkg/models/database/fhir_claim_response.go +++ b/backend/pkg/models/database/fhir_claim_response.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -227,9 +228,14 @@ func (s *FhirClaimResponse) PopulateAndExtractSearchParameters(resourceRaw json. s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting Use useResult, err := vm.RunString("extractTokenSearchParameters(fhirResource, 'ClaimResponse.use')") diff --git a/backend/pkg/models/database/fhir_composition.go b/backend/pkg/models/database/fhir_composition.go index 98329414..dd79d7ec 100644 --- a/backend/pkg/models/database/fhir_composition.go +++ b/backend/pkg/models/database/fhir_composition.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -349,9 +350,14 @@ func (s *FhirComposition) PopulateAndExtractSearchParameters(resourceRaw json.Ra s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting Title titleResult, err := vm.RunString("extractStringSearchParameters(fhirResource, 'Composition.title')") diff --git a/backend/pkg/models/database/fhir_condition.go b/backend/pkg/models/database/fhir_condition.go index 12a01068..ffbdaca2 100644 --- a/backend/pkg/models/database/fhir_condition.go +++ b/backend/pkg/models/database/fhir_condition.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -375,9 +376,14 @@ func (s *FhirCondition) PopulateAndExtractSearchParameters(resourceRaw json.RawM s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting VerificationStatus verificationStatusResult, err := vm.RunString("extractTokenSearchParameters(fhirResource, 'Condition.verificationStatus')") diff --git a/backend/pkg/models/database/fhir_consent.go b/backend/pkg/models/database/fhir_consent.go index 9136b58c..1e6e2278 100644 --- a/backend/pkg/models/database/fhir_consent.go +++ b/backend/pkg/models/database/fhir_consent.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -321,9 +322,14 @@ func (s *FhirConsent) PopulateAndExtractSearchParameters(resourceRaw json.RawMes s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_coverage.go b/backend/pkg/models/database/fhir_coverage.go index 2cf3d493..1eb4f460 100644 --- a/backend/pkg/models/database/fhir_coverage.go +++ b/backend/pkg/models/database/fhir_coverage.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -207,9 +208,14 @@ func (s *FhirCoverage) PopulateAndExtractSearchParameters(resourceRaw json.RawMe s.Subscriber = []byte(subscriberResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_coverage_eligibility_request.go b/backend/pkg/models/database/fhir_coverage_eligibility_request.go index 2e52f702..afada40a 100644 --- a/backend/pkg/models/database/fhir_coverage_eligibility_request.go +++ b/backend/pkg/models/database/fhir_coverage_eligibility_request.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -188,9 +189,14 @@ func (s *FhirCoverageEligibilityRequest) PopulateAndExtractSearchParameters(reso s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_coverage_eligibility_response.go b/backend/pkg/models/database/fhir_coverage_eligibility_response.go index afb6dcbe..f0f87194 100644 --- a/backend/pkg/models/database/fhir_coverage_eligibility_response.go +++ b/backend/pkg/models/database/fhir_coverage_eligibility_response.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -206,9 +207,14 @@ func (s *FhirCoverageEligibilityResponse) PopulateAndExtractSearchParameters(res s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_device.go b/backend/pkg/models/database/fhir_device.go index f765bb05..773807fd 100644 --- a/backend/pkg/models/database/fhir_device.go +++ b/backend/pkg/models/database/fhir_device.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -201,9 +202,14 @@ func (s *FhirDevice) PopulateAndExtractSearchParameters(resourceRaw json.RawMess s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting UdiCarrier udiCarrierResult, err := vm.RunString("extractStringSearchParameters(fhirResource, 'Device.udiCarrier.carrierHRF')") diff --git a/backend/pkg/models/database/fhir_device_request.go b/backend/pkg/models/database/fhir_device_request.go index 0633c5cf..027fbda6 100644 --- a/backend/pkg/models/database/fhir_device_request.go +++ b/backend/pkg/models/database/fhir_device_request.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -360,9 +361,14 @@ func (s *FhirDeviceRequest) PopulateAndExtractSearchParameters(resourceRaw json. s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_diagnostic_report.go b/backend/pkg/models/database/fhir_diagnostic_report.go index 36fee357..08517833 100644 --- a/backend/pkg/models/database/fhir_diagnostic_report.go +++ b/backend/pkg/models/database/fhir_diagnostic_report.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -362,9 +363,14 @@ func (s *FhirDiagnosticReport) PopulateAndExtractSearchParameters(resourceRaw js s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_document_manifest.go b/backend/pkg/models/database/fhir_document_manifest.go index 92ae48fa..91c666b8 100644 --- a/backend/pkg/models/database/fhir_document_manifest.go +++ b/backend/pkg/models/database/fhir_document_manifest.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -266,9 +267,14 @@ func (s *FhirDocumentManifest) PopulateAndExtractSearchParameters(resourceRaw js s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_document_reference.go b/backend/pkg/models/database/fhir_document_reference.go index a9f9e42f..c9f63843 100644 --- a/backend/pkg/models/database/fhir_document_reference.go +++ b/backend/pkg/models/database/fhir_document_reference.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -379,9 +380,14 @@ func (s *FhirDocumentReference) PopulateAndExtractSearchParameters(resourceRaw j s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_encounter.go b/backend/pkg/models/database/fhir_encounter.go index 430202af..198765c4 100644 --- a/backend/pkg/models/database/fhir_encounter.go +++ b/backend/pkg/models/database/fhir_encounter.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -384,9 +385,14 @@ func (s *FhirEncounter) PopulateAndExtractSearchParameters(resourceRaw json.RawM s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_endpoint.go b/backend/pkg/models/database/fhir_endpoint.go index 37295114..7725fe12 100644 --- a/backend/pkg/models/database/fhir_endpoint.go +++ b/backend/pkg/models/database/fhir_endpoint.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -180,9 +181,14 @@ func (s *FhirEndpoint) PopulateAndExtractSearchParameters(resourceRaw json.RawMe s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_enrollment_request.go b/backend/pkg/models/database/fhir_enrollment_request.go index 36aa152b..c3202431 100644 --- a/backend/pkg/models/database/fhir_enrollment_request.go +++ b/backend/pkg/models/database/fhir_enrollment_request.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -153,9 +154,14 @@ func (s *FhirEnrollmentRequest) PopulateAndExtractSearchParameters(resourceRaw j s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_enrollment_response.go b/backend/pkg/models/database/fhir_enrollment_response.go index 1aae1f25..0586070d 100644 --- a/backend/pkg/models/database/fhir_enrollment_response.go +++ b/backend/pkg/models/database/fhir_enrollment_response.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -153,9 +154,14 @@ func (s *FhirEnrollmentResponse) PopulateAndExtractSearchParameters(resourceRaw s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_explanation_of_benefit.go b/backend/pkg/models/database/fhir_explanation_of_benefit.go index a4a0c23d..2fe09056 100644 --- a/backend/pkg/models/database/fhir_explanation_of_benefit.go +++ b/backend/pkg/models/database/fhir_explanation_of_benefit.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -278,9 +279,14 @@ func (s *FhirExplanationOfBenefit) PopulateAndExtractSearchParameters(resourceRa s.SubdetailUdi = []byte(subdetailUdiResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_family_member_history.go b/backend/pkg/models/database/fhir_family_member_history.go index fe2a17bf..b3eb50f3 100644 --- a/backend/pkg/models/database/fhir_family_member_history.go +++ b/backend/pkg/models/database/fhir_family_member_history.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -276,9 +277,14 @@ func (s *FhirFamilyMemberHistory) PopulateAndExtractSearchParameters(resourceRaw s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_goal.go b/backend/pkg/models/database/fhir_goal.go index e01165f6..bfe6328a 100644 --- a/backend/pkg/models/database/fhir_goal.go +++ b/backend/pkg/models/database/fhir_goal.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -238,9 +239,14 @@ func (s *FhirGoal) PopulateAndExtractSearchParameters(resourceRaw json.RawMessag } } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_imaging_study.go b/backend/pkg/models/database/fhir_imaging_study.go index b2127e35..5cff30f2 100644 --- a/backend/pkg/models/database/fhir_imaging_study.go +++ b/backend/pkg/models/database/fhir_imaging_study.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -311,9 +312,14 @@ func (s *FhirImagingStudy) PopulateAndExtractSearchParameters(resourceRaw json.R s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_immunization.go b/backend/pkg/models/database/fhir_immunization.go index 3e449499..6194a6d3 100644 --- a/backend/pkg/models/database/fhir_immunization.go +++ b/backend/pkg/models/database/fhir_immunization.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -325,9 +326,14 @@ func (s *FhirImmunization) PopulateAndExtractSearchParameters(resourceRaw json.R s.TargetDisease = []byte(targetDiseaseResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting VaccineCode vaccineCodeResult, err := vm.RunString("extractTokenSearchParameters(fhirResource, 'Immunization.vaccineCode')") diff --git a/backend/pkg/models/database/fhir_insurance_plan.go b/backend/pkg/models/database/fhir_insurance_plan.go index ad21de1a..febc37e2 100644 --- a/backend/pkg/models/database/fhir_insurance_plan.go +++ b/backend/pkg/models/database/fhir_insurance_plan.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -243,9 +244,14 @@ func (s *FhirInsurancePlan) PopulateAndExtractSearchParameters(resourceRaw json. s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_location.go b/backend/pkg/models/database/fhir_location.go index b3d78a22..86aa3cbb 100644 --- a/backend/pkg/models/database/fhir_location.go +++ b/backend/pkg/models/database/fhir_location.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -243,9 +244,14 @@ func (s *FhirLocation) PopulateAndExtractSearchParameters(resourceRaw json.RawMe s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_media.go b/backend/pkg/models/database/fhir_media.go index 378d32a5..2376e4b6 100644 --- a/backend/pkg/models/database/fhir_media.go +++ b/backend/pkg/models/database/fhir_media.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -228,9 +229,14 @@ func (s *FhirMedia) PopulateAndExtractSearchParameters(resourceRaw json.RawMessa s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting View viewResult, err := vm.RunString("extractTokenSearchParameters(fhirResource, 'Media.view')") diff --git a/backend/pkg/models/database/fhir_medication.go b/backend/pkg/models/database/fhir_medication.go index 78d8914f..2028132d 100644 --- a/backend/pkg/models/database/fhir_medication.go +++ b/backend/pkg/models/database/fhir_medication.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -232,9 +233,14 @@ func (s *FhirMedication) PopulateAndExtractSearchParameters(resourceRaw json.Raw s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_medication_administration.go b/backend/pkg/models/database/fhir_medication_administration.go index d9534b8c..17939018 100644 --- a/backend/pkg/models/database/fhir_medication_administration.go +++ b/backend/pkg/models/database/fhir_medication_administration.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -306,9 +307,14 @@ func (s *FhirMedicationAdministration) PopulateAndExtractSearchParameters(resour s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_medication_dispense.go b/backend/pkg/models/database/fhir_medication_dispense.go index 7cf72bf3..e4381e53 100644 --- a/backend/pkg/models/database/fhir_medication_dispense.go +++ b/backend/pkg/models/database/fhir_medication_dispense.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -301,9 +302,14 @@ func (s *FhirMedicationDispense) PopulateAndExtractSearchParameters(resourceRaw s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting Whenhandedover whenhandedoverResult, err := vm.RunString("extractDateSearchParameters(fhirResource, 'MedicationDispense.whenHandedOver')") diff --git a/backend/pkg/models/database/fhir_medication_request.go b/backend/pkg/models/database/fhir_medication_request.go index 9be6515a..3f891e1d 100644 --- a/backend/pkg/models/database/fhir_medication_request.go +++ b/backend/pkg/models/database/fhir_medication_request.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -349,9 +350,14 @@ func (s *FhirMedicationRequest) PopulateAndExtractSearchParameters(resourceRaw j s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_medication_statement.go b/backend/pkg/models/database/fhir_medication_statement.go index 54dc931c..0f0ea8e5 100644 --- a/backend/pkg/models/database/fhir_medication_statement.go +++ b/backend/pkg/models/database/fhir_medication_statement.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -288,9 +289,14 @@ func (s *FhirMedicationStatement) PopulateAndExtractSearchParameters(resourceRaw s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_nutrition_order.go b/backend/pkg/models/database/fhir_nutrition_order.go index 20cd82fa..a058531b 100644 --- a/backend/pkg/models/database/fhir_nutrition_order.go +++ b/backend/pkg/models/database/fhir_nutrition_order.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -281,9 +282,14 @@ func (s *FhirNutritionOrder) PopulateAndExtractSearchParameters(resourceRaw json s.Supplement = []byte(supplementResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_observation.go b/backend/pkg/models/database/fhir_observation.go index 61afb41e..4881b521 100644 --- a/backend/pkg/models/database/fhir_observation.go +++ b/backend/pkg/models/database/fhir_observation.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -460,9 +461,14 @@ func (s *FhirObservation) PopulateAndExtractSearchParameters(resourceRaw json.Ra s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting ValueConcept valueConceptResult, err := vm.RunString("extractTokenSearchParameters(fhirResource, '(Observation.valueCodeableConcept)')") diff --git a/backend/pkg/models/database/fhir_organization.go b/backend/pkg/models/database/fhir_organization.go index 990dde24..1840a8bb 100644 --- a/backend/pkg/models/database/fhir_organization.go +++ b/backend/pkg/models/database/fhir_organization.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -234,9 +235,14 @@ func (s *FhirOrganization) PopulateAndExtractSearchParameters(resourceRaw json.R s.Phonetic = []byte(phoneticResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_organization_affiliation.go b/backend/pkg/models/database/fhir_organization_affiliation.go index 095dfcf3..9536f925 100644 --- a/backend/pkg/models/database/fhir_organization_affiliation.go +++ b/backend/pkg/models/database/fhir_organization_affiliation.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -260,9 +261,14 @@ func (s *FhirOrganizationAffiliation) PopulateAndExtractSearchParameters(resourc s.Telecom = []byte(telecomResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_patient.go b/backend/pkg/models/database/fhir_patient.go index 71ceb6b5..4eff01cd 100644 --- a/backend/pkg/models/database/fhir_patient.go +++ b/backend/pkg/models/database/fhir_patient.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -436,9 +437,14 @@ func (s *FhirPatient) PopulateAndExtractSearchParameters(resourceRaw json.RawMes s.Telecom = []byte(telecomResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_person.go b/backend/pkg/models/database/fhir_person.go index f41d335f..3a62a88e 100644 --- a/backend/pkg/models/database/fhir_person.go +++ b/backend/pkg/models/database/fhir_person.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -382,9 +383,14 @@ func (s *FhirPerson) PopulateAndExtractSearchParameters(resourceRaw json.RawMess s.Telecom = []byte(telecomResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_practitioner.go b/backend/pkg/models/database/fhir_practitioner.go index 46540dbf..dd87018d 100644 --- a/backend/pkg/models/database/fhir_practitioner.go +++ b/backend/pkg/models/database/fhir_practitioner.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -369,9 +370,14 @@ func (s *FhirPractitioner) PopulateAndExtractSearchParameters(resourceRaw json.R s.Telecom = []byte(telecomResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_practitioner_role.go b/backend/pkg/models/database/fhir_practitioner_role.go index 703a43fb..b38f5108 100644 --- a/backend/pkg/models/database/fhir_practitioner_role.go +++ b/backend/pkg/models/database/fhir_practitioner_role.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -275,9 +276,14 @@ func (s *FhirPractitionerRole) PopulateAndExtractSearchParameters(resourceRaw js s.Telecom = []byte(telecomResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_procedure.go b/backend/pkg/models/database/fhir_procedure.go index 09774e84..3565c5ca 100644 --- a/backend/pkg/models/database/fhir_procedure.go +++ b/backend/pkg/models/database/fhir_procedure.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -354,9 +355,14 @@ func (s *FhirProcedure) PopulateAndExtractSearchParameters(resourceRaw json.RawM s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_provenance.go b/backend/pkg/models/database/fhir_provenance.go index ea9a3a6c..1cbd7b01 100644 --- a/backend/pkg/models/database/fhir_provenance.go +++ b/backend/pkg/models/database/fhir_provenance.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -210,9 +211,14 @@ func (s *FhirProvenance) PopulateAndExtractSearchParameters(resourceRaw json.Raw s.Target = []byte(targetResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting When whenResult, err := vm.RunString("extractDateSearchParameters(fhirResource, '(Provenance.occurredDateTime)')") diff --git a/backend/pkg/models/database/fhir_questionnaire.go b/backend/pkg/models/database/fhir_questionnaire.go index 0cb652fe..868c12a8 100644 --- a/backend/pkg/models/database/fhir_questionnaire.go +++ b/backend/pkg/models/database/fhir_questionnaire.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -280,9 +281,14 @@ func (s *FhirQuestionnaire) PopulateAndExtractSearchParameters(resourceRaw json. s.SubjectType = []byte(subjectTypeResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } // extracting Title titleResult, err := vm.RunString("extractStringSearchParameters(fhirResource, 'Questionnaire.title')") diff --git a/backend/pkg/models/database/fhir_questionnaire_response.go b/backend/pkg/models/database/fhir_questionnaire_response.go index 9b3fa91c..2f5d1b43 100644 --- a/backend/pkg/models/database/fhir_questionnaire_response.go +++ b/backend/pkg/models/database/fhir_questionnaire_response.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -224,9 +225,14 @@ func (s *FhirQuestionnaireResponse) PopulateAndExtractSearchParameters(resourceR s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_related_person.go b/backend/pkg/models/database/fhir_related_person.go index ade88f29..ff675f9d 100644 --- a/backend/pkg/models/database/fhir_related_person.go +++ b/backend/pkg/models/database/fhir_related_person.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -364,9 +365,14 @@ func (s *FhirRelatedPerson) PopulateAndExtractSearchParameters(resourceRaw json. s.Telecom = []byte(telecomResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_schedule.go b/backend/pkg/models/database/fhir_schedule.go index fd7b4560..9dac32bd 100644 --- a/backend/pkg/models/database/fhir_schedule.go +++ b/backend/pkg/models/database/fhir_schedule.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -197,9 +198,14 @@ func (s *FhirSchedule) PopulateAndExtractSearchParameters(resourceRaw json.RawMe s.Specialty = []byte(specialtyResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_service_request.go b/backend/pkg/models/database/fhir_service_request.go index 535d4628..1a4a2699 100644 --- a/backend/pkg/models/database/fhir_service_request.go +++ b/backend/pkg/models/database/fhir_service_request.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -387,9 +388,14 @@ func (s *FhirServiceRequest) PopulateAndExtractSearchParameters(resourceRaw json s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_slot.go b/backend/pkg/models/database/fhir_slot.go index 4d5701b6..99edc684 100644 --- a/backend/pkg/models/database/fhir_slot.go +++ b/backend/pkg/models/database/fhir_slot.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -206,9 +207,14 @@ func (s *FhirSlot) PopulateAndExtractSearchParameters(resourceRaw json.RawMessag s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_specimen.go b/backend/pkg/models/database/fhir_specimen.go index 992aca68..2d6057c7 100644 --- a/backend/pkg/models/database/fhir_specimen.go +++ b/backend/pkg/models/database/fhir_specimen.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -224,9 +225,14 @@ func (s *FhirSpecimen) PopulateAndExtractSearchParameters(resourceRaw json.RawMe s.Subject = []byte(subjectResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/fhir_vision_prescription.go b/backend/pkg/models/database/fhir_vision_prescription.go index 902578eb..83078e57 100644 --- a/backend/pkg/models/database/fhir_vision_prescription.go +++ b/backend/pkg/models/database/fhir_vision_prescription.go @@ -6,6 +6,7 @@ package database import ( "encoding/json" "fmt" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown" goja "github.com/dop251/goja" models "github.com/fastenhealth/fasten-onprem/backend/pkg/models" datatypes "gorm.io/datatypes" @@ -227,9 +228,14 @@ func (s *FhirVisionPrescription) PopulateAndExtractSearchParameters(resourceRaw s.Status = []byte(statusResult.String()) } // extracting Text - textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text')") + textResult, err := vm.RunString("extractSimpleSearchParameters(fhirResource, 'text.div')") if err == nil && textResult.String() != "undefined" { s.Text = textResult.String() + converter := htmltomarkdown.NewConverter("", true, nil) + markdown, err := converter.ConvertString(s.Text) + if err == nil { + s.Text = markdown + } } return nil } diff --git a/backend/pkg/models/database/generate.go b/backend/pkg/models/database/generate.go index 20e41d7b..04f60bdf 100644 --- a/backend/pkg/models/database/generate.go +++ b/backend/pkg/models/database/generate.go @@ -168,7 +168,7 @@ func main() { fieldMap["Text"] = DBField{ FieldType: "keyword", Description: "Text search against the narrative", - FHIRPathExpression: "text", + FHIRPathExpression: "text.div", } fieldMap["Type"] = DBField{ FieldType: "special", @@ -424,6 +424,14 @@ func main() { break default: i.Id("s").Dot(fieldName).Op("=").Id(fieldNameVar).Dot("String").Call() + if fieldName == "Text" { + //convert html to markdown + i.Id("converter").Op(":=").Qual("github.com/JohannesKaufmann/html-to-markdown", "NewConverter").Call(jen.Lit(""), jen.True(), jen.Nil()) + i.List(jen.Id("markdown"), jen.Id("err")).Op(":=").Id("converter").Dot("ConvertString").Call(jen.Id("s").Dot(fieldName)) + i.If(jen.Err().Op("==").Nil()).BlockFunc(func(q *jen.Group) { + q.Id("s").Dot(fieldName).Op("=").Id("markdown") + }) + } break } diff --git a/go.mod b/go.mod index 2c15293a..9ed14fa4 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ replace github.com/mattn/go-sqlite3 v1.14.17 => github.com/jgiannuzzi/go-sqlite3 //replace gorm.io/driver/sqlite v1.5.4 => github.com/jgiannuzzi/gorm-sqlite v1.4.4-0.20221215225833-42389ad31305 require ( + github.com/JohannesKaufmann/html-to-markdown v1.5.0 github.com/analogj/go-util v0.0.0-20210417161720-39b497cca03b github.com/dave/jennifer v1.6.1 github.com/dominikbraun/graph v0.15.0 @@ -30,16 +31,18 @@ require ( github.com/spf13/viper v1.12.0 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.11.2 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.16.0 golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 golang.org/x/mod v0.15.0 - golang.org/x/net v0.17.0 + golang.org/x/net v0.19.0 gorm.io/datatypes v1.0.7 gorm.io/driver/sqlite v1.5.4 gorm.io/gorm v1.25.4 ) require ( + github.com/PuerkitoBio/goquery v1.8.1 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect @@ -103,9 +106,9 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect diff --git a/go.sum b/go.sum index 5e17611d..152ba002 100644 --- a/go.sum +++ b/go.sum @@ -41,11 +41,17 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJc github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/JohannesKaufmann/html-to-markdown v1.5.0 h1:cEAcqpxk0hUJOXEVGrgILGW76d1GpyGY7PCnAaWQyAI= +github.com/JohannesKaufmann/html-to-markdown v1.5.0/go.mod h1:QTO/aTyEDukulzu269jY0xiHeAGsNxmuUBo2Q0hPsK8= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/analogj/go-util v0.0.0-20210417161720-39b497cca03b h1:Y/+MfmdKPPpVY7C6ggt/FpltFSitlpUtyJEdcQyFXQg= github.com/analogj/go-util v0.0.0-20210417161720-39b497cca03b/go.mod h1:bRSzJXgXnT5+Ihah7RSC7Cvp16UmoLn3wq6ROciS1Ow= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.8.8 h1:Kj4AYbZSeENfyXicsYppYKO0K2YWab+i2UTSY7Ukz9Q= @@ -365,10 +371,15 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/samber/lo v1.35.0 h1:GlT8CV1GE+v97Y7MLF1wXvX6mjoxZ+hi61tj/ZcQwY0= github.com/samber/lo v1.35.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= +github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/seborama/govcr v4.5.0+incompatible h1:XvdHtXi0d4cUAn+0aWolvwfS3nmhNC8Z+yMQwn/M64I= github.com/seborama/govcr v4.5.0+incompatible/go.mod h1:EgcISudCCYDLzbiAImJ8i7kk4+wTA44Kp+j4S0LhASI= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -419,6 +430,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.6.0 h1:boZcn2GTjpsynOsC0iJHnBWa4Bi0qzfJjthwauItG68= +github.com/yuin/goldmark v1.6.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -457,8 +470,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -533,12 +546,14 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -620,15 +635,15 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -640,8 +655,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -801,6 +816,7 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -808,6 +824,7 @@ gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:a gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=