adding support for infinite scroll

removed fuse (client side search) replaced with remote search support.

frontend/package.json

@@ -36,7 +36,6 @@
     "bootstrap": "^4.4.1",
     "chart.js": "2.9.4",
     "fhirpath": "^3.3.0",
-    "fuse.js": "^6.6.2",
     "humanize-duration": "^3.27.3",
     "idb": "^7.1.0",
     "jose": "^4.10.4",
@@ -44,6 +43,7 @@
     "ng2-charts": "^2.3.0",
     "ngx-dropzone": "^3.1.0",
     "ngx-highlightjs": "^7.0.1",
+    "ngx-infinite-scroll": "^14.0.0",
     "ngx-moment": "^6.0.2",
     "rxjs": "~6.5.4",
     "tslib": "^2.0.0",

frontend/src/app/app.module.ts

@@ -30,7 +30,7 @@ import { MedicalHistoryComponent } from './pages/medical-history/medical-history
 import { ReportLabsComponent } from './pages/report-labs/report-labs.component';
 import {PipesModule} from './pipes/pipes.module';
 import { ResourceCreatorComponent } from './pages/resource-creator/resource-creator.component';
-
+import { InfiniteScrollModule } from 'ngx-infinite-scroll';
 
 @NgModule({
   declarations: [
@@ -62,6 +62,7 @@ import { ResourceCreatorComponent } from './pages/resource-creator/resource-crea
     HighlightModule,
     MomentModule,
     PipesModule,
+    InfiniteScrollModule
   ],
   providers: [
     {

frontend/src/app/models/lighthouse/lighthouse-source-metadata.ts

@@ -1,4 +1,7 @@
-export class LighthouseSourceMetadata {
+import {MetadataSource} from '../fasten/metadata-source';
+
+export class LighthouseSourceMetadata extends MetadataSource {
+
   authorization_endpoint: string
   token_endpoint: string
   introspection_endpoint: string

frontend/src/app/models/lighthouse/lighthouse-source-search.ts

@@ -0,0 +1,21 @@
+import {MetadataSource} from '../fasten/metadata-source';
+
+export class LighthouseSourceSearch {
+  _scroll_id: string;
+  took: number;
+  timed_out: boolean;
+  hits: {
+    total: {
+      value: number;
+      relation: string;
+    };
+    max_score: number;
+    hits: {
+      _index: string;
+      _type: string;
+      _id: string;
+      _score: number;
+      _source: MetadataSource;
+    }[];
+  };
+}

frontend/src/app/pages/medical-sources/medical-sources.component.html

@@ -87,18 +87,23 @@
       </div>
 
 
-      <div *ngIf="!loading else isLoadingTemplate" class="row row-sm">
+      <div *ngIf="!loading else isLoadingTemplate" class="row row-sm"
+           infiniteScroll
+           [infiniteScrollDistance]="2"
+           [infiniteScrollThrottle]="50"
+           (scrolled)="onScroll()"
+      >
         <div *ngFor="let sourceData of availableSourceList" (click)="connectHandler($event, sourceData.metadata.source_type)" class="col-sm-3 mg-b-20 px-3">
           <div class="card h-100 d-flex align-items-center justify-content-center mt-3 mt-3 rounded-0 cursor-pointer" [ngClass]="{'card-disable': sourceData.metadata.hidden}">
             <div class="card-body d-flex align-items-center">
-              <img style="max-height: 130px;" [src]="'assets/sources/'+sourceData.metadata.source_type+'.png'" [alt]="metadataSources[sourceData.metadata.source_type].display" class="img-fluid">
+              <img style="max-height: 130px;" [src]="'assets/sources/'+sourceData.metadata.source_type+'.png'" [alt]="sourceData.metadata.display" class="img-fluid">
               <div *ngIf="status[sourceData.metadata.source_type]" class="progress">
                 <div [style.width]="status[sourceData.metadata.source_type] == 'authorize' ? '33%' : '66%'" class="bg-indigo progress-bar progress-bar-striped progress-bar-animated" role="progressbar"></div>
               </div>
             </div>
             <div class="card-footer text-center p-1" style="width:100%">
               <small class="tx-gray-700">
-                {{metadataSources[sourceData.metadata.source_type].display}}
+                {{sourceData.metadata.display}}
               </small>
             </div>
           </div>

frontend/src/app/pages/medical-sources/medical-sources.component.ts

@@ -11,7 +11,7 @@ import {ToastService} from '../../services/toast.service';
 import {ToastNotification, ToastType} from '../../models/fasten/toast';
 import {environment} from '../../../environments/environment';
 import {forkJoin} from 'rxjs';
-import Fuse from 'fuse.js'
+import {LighthouseSourceSearch} from '../../models/lighthouse/lighthouse-source-search';
 // If you dont import this angular will import the wrong "Location"
 
 export const sourceConnectWindowTimeout = 24*5000 //wait 2 minutes (5 * 24 = 120)
@@ -32,8 +32,6 @@ export class MedicalSourcesComponent implements OnInit {
   environment_name = environment.environment_name
   status: { [name: string]: string } = {}
 
-  metadataSources: {[name:string]: MetadataSource} = {}
-
   connectedSourceList: SourceListItem[] = [] //source's are populated for this list
   availableSourceList: SourceListItem[] = []
   totalAvailableSourceList: number = 0
@@ -42,9 +40,10 @@ export class MedicalSourcesComponent implements OnInit {
   closeResult = '';
   modalSelectedSourceListItem:SourceListItem = null;
 
-  searchIndex = null
+  scrollId: string = ""
+  scrollComplete: boolean = false
   searchTerm: string = ""
-
+  showHidden: boolean = false
   constructor(
     private lighthouseApi: LighthouseService,
     private fastenApi: FastenApiService,
@@ -57,29 +56,20 @@ export class MedicalSourcesComponent implements OnInit {
 
   ngOnInit(): void {
     this.loading = true
-    forkJoin([this.lighthouseApi.getLighthouseSourceMetadataMap(false), this.fastenApi.getSources()]).subscribe(results => {
+    forkJoin([this.lighthouseApi.findLighthouseSources("", "", this.showHidden), this.fastenApi.getSources()]).subscribe(results => {
       this.loading = false
+
+      //handle connected sources sources
+      const connectedSources = results[1] as Source[]
+      forkJoin(connectedSources.map((source) => this.lighthouseApi.getLighthouseSource(source.source_type))).subscribe((connectedMetadata) => {
+        for(const ndx in connectedSources){
+          this.connectedSourceList.push({source: connectedSources[ndx], metadata: connectedMetadata[ndx]})
+        }
+      })
+
+
       //handle source metadata map response
-      this.metadataSources = results[0] as {[name:string]: MetadataSource}
+      this.populateAvailableSourceList(results[0] as LighthouseSourceSearch)
-
-      //handle sources
-      const sourceList = results[1] as Source[]
-
-      for (const sourceType in this.metadataSources) {
-        let isConnected = false
-        for(const connectedSource of sourceList){
-          if(connectedSource.source_type == sourceType){
-            this.connectedSourceList.push({source: connectedSource, metadata: this.metadataSources[sourceType]})
-            isConnected = true
-            break
-          }
-        }
-
-        if(!isConnected){
-          //this source has not been found in the connected list, lets add it to the available list.
-          this.availableSourceList.push({metadata: this.metadataSources[sourceType]})
-        }
-      }
 
 
       //check if we've just started connecting a "source_type"
@@ -97,55 +87,55 @@ export class MedicalSourcesComponent implements OnInit {
 
         this.callback(callbackSourceType).then(console.log)
       }
-      this.totalAvailableSourceList = this.availableSourceList.length
-
-      //setup Search
-      const options = {
-        // isCaseSensitive: false,
-        // includeScore: false,
-        // shouldSort: true,
-        // includeMatches: false,
-        findAllMatches: true,
-        // minMatchCharLength: 1,
-        // location: 0,
-        // threshold: 0.6,
-        // distance: 100,
-        // useExtendedSearch: false,
-        // ignoreLocation: false,
-        // ignoreFieldNorm: false,
-        // fieldNormWeight: 1,
-        keys: [
-          "metadata.display",
-          "metadata.category",
-          "metadata.source_type"
-        ]
-      };
-
-      this.searchIndex = new Fuse(this.availableSourceList, options);
-
 
     }, err => {
       this.loading = false
     })
   }
 
+  private populateAvailableSourceList(results: LighthouseSourceSearch): void {
+    this.totalAvailableSourceList = results.hits.total.value
+    if(results.hits.hits.length == 0){
+      this.scrollComplete = true
+      console.log("scroll complete")
+      return
+    }
+    this.scrollId = results._scroll_id
+    this.availableSourceList = this.availableSourceList.concat(results.hits.hits.map((result) => {
+      return {metadata: result._source}
+    }).filter((item) => {
+      return !this.connectedSourceList.find((connectedItem) => connectedItem.metadata.source_type == item.metadata.source_type)
+    }))
+  }
+
+  public onScroll(): void {
+    if(this.scrollComplete){
+      return
+    }
+
+    this.lighthouseApi.findLighthouseSources(this.searchTerm, this.scrollId, this.showHidden)
+      .subscribe((results) => {
+        this.populateAvailableSourceList(results)
+      })
+  }
+
   public searchTermChanged($event):void {
     this.searchTerm = $event.target.value
     console.log("search term changed:", )
 
-    let searchResults
+    // let searchResults
-    if(this.searchTerm){
+    // if(this.searchTerm){
-      searchResults = this.searchIndex.search(this.searchTerm).map((result) => {
+    //   searchResults = this.searchIndex.search(this.searchTerm).map((result) => {
-        return result.item
+    //     return result.item
-      })
+    //   })
-    }
+    // }
-    else {
+    // else {
-      //emtpy search term, show all (original) values.
+    //   //emtpy search term, show all (original) values.
-      searchResults = this.searchIndex.getIndex().docs
+    //   searchResults = this.searchIndex.getIndex().docs
-    }
+    // }
-
+    //
-    this.availableSourceList = searchResults
+    // this.availableSourceList = searchResults
-    console.log(this.availableSourceList)
+    // console.log(this.availableSourceList)
   }
 
   /**

frontend/src/app/services/lighthouse.service.ts

@@ -9,6 +9,7 @@ import * as Oauth from '@panva/oauth4webapi';
 import {SourceState} from '../models/fasten/source-state';
 import {MetadataSource} from '../models/fasten/metadata-source';
 import {uuidV4} from '../../lib/utils/uuid';
+import {LighthouseSourceSearch} from '../models/lighthouse/lighthouse-source-search';
 
 @Injectable({
   providedIn: 'root'
@@ -18,6 +19,24 @@ export class LighthouseService {
   constructor(private _httpClient: HttpClient) {
   }
 
+  public findLighthouseSources(searchTerm: string, scrollId= "", showHidden = false): Observable<LighthouseSourceSearch> {
+    const endpointUrl = new URL(`${environment.lighthouse_api_endpoint_base}/list/search`);
+    if(showHidden){
+      endpointUrl.searchParams.set('show_hidden', 'true');
+    }
+    if(scrollId){
+      endpointUrl.searchParams.set('scroll_id', scrollId);
+    }
+
+    return this._httpClient.get<ResponseWrapper>(endpointUrl.toString())
+      .pipe(
+        map((response: ResponseWrapper) => {
+          console.log("Metadata RESPONSE", response)
+          return response.data as LighthouseSourceSearch
+        })
+      );
+  }
+
   public getLighthouseSourceMetadataMap(showHidden = false): Observable<{[name: string]: MetadataSource}> {
     const endpointUrl = new URL(`${environment.lighthouse_api_endpoint_base}/list`);
     if(showHidden){

frontend/yarn.lock

@@ -3869,11 +3869,6 @@ function-bind@^1.1.1:
   resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
   integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==
 
-fuse.js@^6.6.2:
-  version "6.6.2"
-  resolved "https://registry.npmjs.org/fuse.js/-/fuse.js-6.6.2.tgz#fe463fed4b98c0226ac3da2856a415576dc9a111"
-  integrity sha512-cJaJkxCCxC8qIIcPBF9yGxY0W/tVZS3uEISDxhYIdtk8OL93pe+6Zj7LjCqVV4dzbqcriOZ+kQ/NE4RXZHsIGA==
-
 gauge@^4.0.3:
   version "4.0.4"
   resolved "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz#52ff0652f2bbf607a989793d53b751bef2328dce"
@@ -5291,6 +5286,13 @@ ngx-highlightjs@^7.0.1:
     highlightjs-line-numbers.js "^2.8.0"
     tslib "^2.0.0"
 
+ngx-infinite-scroll@^14.0.0:
+  version "14.0.1"
+  resolved "https://registry.npmjs.org/ngx-infinite-scroll/-/ngx-infinite-scroll-14.0.1.tgz#037abd55c441300651d03bc64a69d3475cce7bf9"
+  integrity sha512-PlGL29d2PxNJTn6qdXs4Es0HlJTZ/ZqOVvFWECWm7mK2fN/q+q62s0iUQ7xRf76NuqoNovXvrjZ1zwLFT6c0Wg==
+  dependencies:
+    tslib "^2.3.0"
+
 ngx-moment@^6.0.2:
   version "6.0.2"
   resolved "https://registry.npmjs.org/ngx-moment/-/ngx-moment-6.0.2.tgz#7acba9830746e9c2862d261150f2c8adb20eb4f6"