working aggregation code.

added tests for sql generation.
This commit is contained in:
Jason Kulatunga 2023-08-14 16:50:09 -06:00
parent 2c5133bef7
commit 6f56fa9c3b
4 changed files with 413 additions and 7 deletions

View File

@ -24,8 +24,8 @@ const (
SearchParameterTypeToken SearchParameterType = "token"
SearchParameterTypeReference SearchParameterType = "reference"
SearchParameterTypeUri SearchParameterType = "uri"
SearchParameterTypeComposite SearchParameterType = "composite"
SearchParameterTypeQuantity SearchParameterType = "quantity"
SearchParameterTypeComposite SearchParameterType = "composite"
SearchParameterTypeSpecial SearchParameterType = "special"
)
@ -131,23 +131,67 @@ func (sr *SqliteRepository) sqlQueryResources(ctx context.Context, query models.
//defaults
selectClauses := []string{fmt.Sprintf("%s.*", TABLE_ALIAS)}
groupClause := fmt.Sprintf("%s.id", TABLE_ALIAS)
orderClause := fmt.Sprintf("%s.sort_date asc", TABLE_ALIAS)
orderClause := fmt.Sprintf("%s.sort_date ASC", TABLE_ALIAS)
if query.Aggregations != nil {
selectClauses = []string{}
groupClause = ""
orderClause = ""
//Handle Aggregations
if len(query.Aggregations.CountBy) > 0 {
//populate the group by and order by clause with the count by values
query.Aggregations.OrderBy = "count(*) DESC"
query.Aggregations.GroupBy = query.Aggregations.CountBy
}
//TODO:
//process order by clause
if len(query.Aggregations.OrderBy) > 0 {
orderAsc := true
if !strings.HasPrefix(query.Aggregations.OrderBy, "count(*)") {
orderAggregationParam, err := ProcessAggregationParameter(query.Aggregations.OrderBy, searchCodeToTypeLookup)
if err != nil {
return nil, err
}
orderAggregationFromClause, err := SearchCodeToFromClause(orderAggregationParam)
if err != nil {
return nil, err
}
fromClauses = append(fromClauses, orderAggregationFromClause)
orderClause = AggregationParameterToClause(orderAggregationParam)
if orderAsc {
orderClause = fmt.Sprintf("%s ASC", orderClause)
} else {
orderClause = fmt.Sprintf("%s DESC", orderClause)
}
} else {
orderClause = query.Aggregations.OrderBy
}
}
//process group by clause
if len(query.Aggregations.GroupBy) > 0 {
groupAggregationParam, err := ProcessAggregationParameter(query.Aggregations.GroupBy, searchCodeToTypeLookup)
if err != nil {
return nil, err
}
groupAggregationFromClause, err := SearchCodeToFromClause(groupAggregationParam)
if err != nil {
return nil, err
}
fromClauses = append(fromClauses, groupAggregationFromClause)
groupClause = AggregationParameterToClause(groupAggregationParam)
selectClauses = []string{
fmt.Sprintf("%s as %s", groupClause, "group_by"),
"count(*) as count",
}
}
}
//ensure Where and From clauses are unique
whereClauses = lo.Uniq(whereClauses)
whereClauses = lo.Compact(whereClauses)
fromClauses = lo.Uniq(fromClauses)
fromClauses = lo.Compact(fromClauses)
return sr.GormClient.WithContext(ctx).
Select(strings.Join(selectClauses, ", ")).
@ -158,6 +202,7 @@ func (sr *SqliteRepository) sqlQueryResources(ctx context.Context, query models.
}
/// INTERNAL functionality. These functions are exported for testing, but are not available in the Interface
//TODO: dont export these, instead use casting to convert the interface to the SqliteRepository struct, then call ehese functions directly
type SearchParameter struct {
Name string
@ -499,3 +544,56 @@ func SearchCodeToFromClause(searchParam SearchParameter) (string, error) {
}
return "", nil
}
func AggregationParameterToClause(aggParameter SearchParameter) string {
switch aggParameter.Type {
case SearchParameterTypeQuantity, SearchParameterTypeToken, SearchParameterTypeString:
//setup the clause
return fmt.Sprintf("(%sJson.value ->> '$.%s')", aggParameter.Name, aggParameter.Modifier)
default:
return fmt.Sprintf("%s.%s", TABLE_ALIAS, aggParameter.Name)
}
}
//ProcessAggregationParameter processes the aggregation parameters which are fields with optional properties:
// Fields that are primitive types (number, uri) must not have any property specified:
// eg. `probability`
//
// Fields that are complex types (token, quantity) must have a property specified:
// eg. `identifier:code`
//
// if the a property is specified, its set as the modifier, and used when generating the SQL query groupBy, orderBy, etc clause
func ProcessAggregationParameter(aggregationFieldWithProperty string, searchParamTypeLookup map[string]string) (SearchParameter, error) {
aggregationParameter := SearchParameter{}
//determine the searchCode searchCodeModifier
//TODO: this is only applicable to string, token, reference and uri type (however unknown names & modifiers are ignored)
if aggregationFieldParts := strings.SplitN(aggregationFieldWithProperty, ":", 2); len(aggregationFieldParts) == 2 {
aggregationParameter.Name = aggregationFieldParts[0]
aggregationParameter.Modifier = aggregationFieldParts[1]
} else {
aggregationParameter.Name = aggregationFieldParts[0]
aggregationParameter.Modifier = ""
}
//next, determine the searchCodeType for this Resource (or throw an error if it is unknown)
searchParamTypeStr, searchParamTypeOk := searchParamTypeLookup[aggregationParameter.Name]
if !searchParamTypeOk {
return aggregationParameter, fmt.Errorf("unknown search parameter: %s", aggregationParameter.Name)
} else {
aggregationParameter.Type = SearchParameterType(searchParamTypeStr)
}
//primitive types should not have a modifier, we need to throw an error
if aggregationParameter.Type == SearchParameterTypeNumber || aggregationParameter.Type == SearchParameterTypeUri {
if len(aggregationParameter.Modifier) > 0 {
return aggregationParameter, fmt.Errorf("primitive aggregation parameter %s cannot have a property (%s)", aggregationParameter.Name, aggregationParameter.Modifier)
}
} else {
//complex types must have a modifier
if len(aggregationParameter.Modifier) == 0 {
return aggregationParameter, fmt.Errorf("complex aggregation parameter %s must have a property", aggregationParameter.Name)
}
}
return aggregationParameter, nil
}

View File

@ -0,0 +1,292 @@
package database
import (
"context"
"fmt"
"github.com/fastenhealth/fastenhealth-onprem/backend/pkg"
mock_config "github.com/fastenhealth/fastenhealth-onprem/backend/pkg/config/mock"
"github.com/fastenhealth/fastenhealth-onprem/backend/pkg/models"
"github.com/golang/mock/gomock"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"gorm.io/gorm"
"io/ioutil"
"log"
"os"
"strings"
"testing"
)
// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context
type RepositorySqlTestSuite struct {
suite.Suite
MockCtrl *gomock.Controller
TestDatabase *os.File
TestRepository DatabaseRepository
}
// BeforeTest has a function to be executed right before the test starts and receives the suite and test names as input
func (suite *RepositorySqlTestSuite) BeforeTest(suiteName, testName string) {
suite.MockCtrl = gomock.NewController(suite.T())
dbFile, err := ioutil.TempFile("", fmt.Sprintf("%s.*.db", testName))
if err != nil {
log.Fatal(err)
}
suite.TestDatabase = dbFile
fakeConfig := mock_config.NewMockInterface(suite.MockCtrl)
fakeConfig.EXPECT().GetString("database.location").Return(suite.TestDatabase.Name()).AnyTimes()
fakeConfig.EXPECT().GetString("log.level").Return("INFO").AnyTimes()
dbRepo, err := NewRepository(fakeConfig, logrus.WithField("test", suite.T().Name()))
require.NoError(suite.T(), err)
suite.TestRepository = dbRepo
userModel := &models.User{
Username: "test_username",
Password: "testpassword",
Email: "test@test.com",
}
err = suite.TestRepository.CreateUser(context.Background(), userModel)
require.NoError(suite.T(), err)
}
// AfterTest has a function to be executed right after the test finishes and receives the suite and test names as input
func (suite *RepositorySqlTestSuite) AfterTest(suiteName, testName string) {
suite.MockCtrl.Finish()
os.Remove(suite.TestDatabase.Name())
}
// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestRepositorySqlTestSuite(t *testing.T) {
suite.Run(t, new(RepositorySqlTestSuite))
}
func (suite *RepositorySqlTestSuite) TestQueryResources_SQL() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
sqliteRepo.GormClient = sqliteRepo.GormClient.Session(&gorm.Session{DryRun: true})
//test
authContext := context.WithValue(context.Background(), pkg.ContextKeyTypeAuthUsername, "test_username")
sqlQuery, err := sqliteRepo.sqlQueryResources(authContext, models.QueryResource{
Select: []string{},
Where: map[string]interface{}{
"code": "test_code",
},
From: "Observation",
})
require.NoError(suite.T(), err)
var results []map[string]interface{}
statement := sqlQuery.Find(&results).Statement
sqlString := statement.SQL.String()
sqlParams := statement.Vars
//assert
require.NoError(suite.T(), err)
require.Equal(suite.T(),
strings.Join([]string{
"SELECT fhir.*",
"FROM fhir_observation as fhir, json_each(fhir.code) as codeJson",
"WHERE ((codeJson.value ->> '$.code' = ?)) AND (user_id = ?)",
"GROUP BY `fhir`.`id`",
"ORDER BY fhir.sort_date ASC",
}, " "),
sqlString)
require.Equal(suite.T(), sqlParams, []interface{}{
"test_code", "00000000-0000-0000-0000-000000000000",
})
}
func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithMultipleWhereConditions() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
sqliteRepo.GormClient = sqliteRepo.GormClient.Session(&gorm.Session{DryRun: true})
//test
authContext := context.WithValue(context.Background(), pkg.ContextKeyTypeAuthUsername, "test_username")
sqlQuery, err := sqliteRepo.sqlQueryResources(authContext, models.QueryResource{
Select: []string{},
Where: map[string]interface{}{
"code": "test_code",
"category": "12345",
},
From: "Observation",
})
require.NoError(suite.T(), err)
var results []map[string]interface{}
statement := sqlQuery.Find(&results).Statement
sqlString := statement.SQL.String()
sqlParams := statement.Vars
//assert
require.NoError(suite.T(), err)
require.Equal(suite.T(),
strings.Join([]string{
"SELECT fhir.*",
"FROM fhir_observation as fhir, json_each(fhir.code) as codeJson",
"WHERE ((codeJson.value ->> '$.code' = ?)) AND (user_id = ?)",
"GROUP BY `fhir`.`id`",
"ORDER BY fhir.sort_date ASC",
}, " "),
sqlString)
require.Equal(suite.T(), sqlParams, []interface{}{
"test_code", "00000000-0000-0000-0000-000000000000",
})
}
func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithPrimitiveOrderByAggregation() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
sqliteRepo.GormClient = sqliteRepo.GormClient.Session(&gorm.Session{DryRun: true})
//test
authContext := context.WithValue(context.Background(), pkg.ContextKeyTypeAuthUsername, "test_username")
sqlQuery, err := sqliteRepo.sqlQueryResources(authContext, models.QueryResource{
Select: []string{},
Where: map[string]interface{}{
"code": "test_code",
},
From: "Observation",
Aggregations: &models.QueryResourceAggregations{OrderBy: "sourceUri"},
})
require.NoError(suite.T(), err)
var results []map[string]interface{}
statement := sqlQuery.Find(&results).Statement
sqlString := statement.SQL.String()
sqlParams := statement.Vars
//assert
require.NoError(suite.T(), err)
require.Equal(suite.T(),
strings.Join([]string{
"SELECT fhir.*",
"FROM fhir_observation as fhir, json_each(fhir.code) as codeJson",
"WHERE ((codeJson.value ->> '$.code' = ?)) AND (user_id = ?)",
"GROUP BY `fhir`.`id`",
"ORDER BY fhir.sourceUri ASC",
}, " "), sqlString)
require.Equal(suite.T(), sqlParams, []interface{}{
"test_code", "00000000-0000-0000-0000-000000000000",
})
}
func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithComplexOrderByAggregation() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
sqliteRepo.GormClient = sqliteRepo.GormClient.Session(&gorm.Session{DryRun: true})
//test
authContext := context.WithValue(context.Background(), pkg.ContextKeyTypeAuthUsername, "test_username")
sqlQuery, err := sqliteRepo.sqlQueryResources(authContext, models.QueryResource{
Select: []string{},
Where: map[string]interface{}{
"code": "test_code",
},
From: "Observation",
Aggregations: &models.QueryResourceAggregations{OrderBy: "valueString:value"},
})
require.NoError(suite.T(), err)
var results []map[string]interface{}
statement := sqlQuery.Find(&results).Statement
sqlString := statement.SQL.String()
sqlParams := statement.Vars
//assert
require.NoError(suite.T(), err)
require.Equal(suite.T(),
strings.Join([]string{
"SELECT fhir.*",
"FROM fhir_observation as fhir, json_each(fhir.code) as codeJson, json_each(fhir.valueString) as valueStringJson",
"WHERE ((codeJson.value ->> '$.code' = ?)) AND (user_id = ?)",
"GROUP BY `fhir`.`id`",
"ORDER BY (valueStringJson.value ->> '$.value') ASC",
}, " "), sqlString)
require.Equal(suite.T(), sqlParams, []interface{}{
"test_code", "00000000-0000-0000-0000-000000000000",
})
}
func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithPrimitiveCountByAggregation() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
sqliteRepo.GormClient = sqliteRepo.GormClient.Session(&gorm.Session{DryRun: true})
//test
authContext := context.WithValue(context.Background(), pkg.ContextKeyTypeAuthUsername, "test_username")
sqlQuery, err := sqliteRepo.sqlQueryResources(authContext, models.QueryResource{
Select: []string{},
Where: map[string]interface{}{
"code": "test_code",
},
From: "Observation",
Aggregations: &models.QueryResourceAggregations{CountBy: "sourceUri"},
})
require.NoError(suite.T(), err)
var results []map[string]interface{}
statement := sqlQuery.Find(&results).Statement
sqlString := statement.SQL.String()
sqlParams := statement.Vars
//assert
require.NoError(suite.T(), err)
require.Equal(suite.T(),
strings.Join([]string{
"SELECT fhir.sourceUri as group_by, count(*) as count",
"FROM fhir_observation as fhir, json_each(fhir.code) as codeJson",
"WHERE ((codeJson.value ->> '$.code' = ?)) AND (user_id = ?)",
"GROUP BY `fhir`.`sourceUri`",
"ORDER BY count(*) DESC",
}, " "), sqlString)
require.Equal(suite.T(), sqlParams, []interface{}{
"test_code", "00000000-0000-0000-0000-000000000000",
})
}
func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithComplexCountByAggregation() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
sqliteRepo.GormClient = sqliteRepo.GormClient.Session(&gorm.Session{DryRun: true})
//test
authContext := context.WithValue(context.Background(), pkg.ContextKeyTypeAuthUsername, "test_username")
sqlQuery, err := sqliteRepo.sqlQueryResources(authContext, models.QueryResource{
Select: []string{},
Where: map[string]interface{}{
"code": "test_code",
},
From: "Observation",
Aggregations: &models.QueryResourceAggregations{CountBy: "code:code"},
})
require.NoError(suite.T(), err)
var results []map[string]interface{}
statement := sqlQuery.Find(&results).Statement
sqlString := statement.SQL.String()
sqlParams := statement.Vars
//assert
require.NoError(suite.T(), err)
require.Equal(suite.T(),
strings.Join([]string{
"SELECT (codeJson.value ->> '$.code') as group_by, count(*) as count",
"FROM fhir_observation as fhir, json_each(fhir.code) as codeJson",
"WHERE ((codeJson.value ->> '$.code' = ?)) AND (user_id = ?)",
"GROUP BY (codeJson.value ->> '$.code')",
"ORDER BY count(*) DESC",
}, " "), sqlString)
require.Equal(suite.T(), sqlParams, []interface{}{
"test_code", "00000000-0000-0000-0000-000000000000",
})
}

View File

@ -1,6 +1,9 @@
package models
import "fmt"
import (
"fmt"
"strings"
)
// maps to frontend/src/app/models/widget/dashboard-widget-query.ts
type QueryResource struct {
@ -44,6 +47,15 @@ func (q *QueryResource) Validate() error {
if len(q.Aggregations.CountBy) == 0 && len(q.Aggregations.OrderBy) == 0 && len(q.Aggregations.GroupBy) == 0 {
return fmt.Errorf("aggregations must have at least one of 'count_by', 'group_by', or 'order_by'")
}
if strings.Contains(q.Aggregations.CountBy, " ") {
return fmt.Errorf("count_by cannot have spaces (or aliases)")
}
if strings.Contains(q.Aggregations.GroupBy, " ") {
return fmt.Errorf("group_by cannot have spaces (or aliases)")
}
if strings.Contains(q.Aggregations.OrderBy, " ") {
return fmt.Errorf("order_by cannot have spaces (or aliases)")
}
}

View File

@ -18,6 +18,10 @@ func TestQueryResource_Validate(t *testing.T) {
{QueryResource{From: "test", Aggregations: &QueryResourceAggregations{CountBy: "test", GroupBy: "test"}}, "cannot use 'count_by' and 'group_by' together", true},
{QueryResource{From: "test", Aggregations: &QueryResourceAggregations{CountBy: "test", OrderBy: "test"}}, "cannot use 'count_by' and 'order_by' together", true},
{QueryResource{From: "test", Aggregations: &QueryResourceAggregations{}}, "aggregations must have at least one of 'count_by', 'group_by', or 'order_by'", true},
{QueryResource{From: "test", Aggregations: &QueryResourceAggregations{CountBy: "test:property"}}, "", false},
{QueryResource{From: "test", Aggregations: &QueryResourceAggregations{CountBy: "test:property as HELLO"}}, "count_by cannot have spaces", true},
{QueryResource{From: "test", Aggregations: &QueryResourceAggregations{GroupBy: "test:property as HELLO"}}, "group_by cannot have spaces", true},
{QueryResource{From: "test", Aggregations: &QueryResourceAggregations{OrderBy: "test:property as HELLO"}}, "order_by cannot have spaces", true},
}
//test && assert