cyberes
/
geoserver
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

continue styling the import

This commit is contained in:
Cyberes 2024-09-12 14:53:40 -06:00
parent fcb13f4842
commit 4be332aff6
8 changed files with 167 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/geo-backend/data/views/import_item.py
View File

@ -74,17 +74,17 @@ def fetch_import_queue(request, item_id):
try:
item = ImportQueue.objects.get(id=item_id)
if item.user_id != request.user.id:
return JsonResponse({'success': False, 'msg': 'not authorized to view this item', 'code': 403}, status=400)
return JsonResponse({'success': False, 'processing': False, 'msg': 'not authorized to view this item', 'code': 403}, status=400)
if not lock_manager.is_locked('data_importqueue', item.id) and (len(item.geofeatures) or len(item.log)):
return JsonResponse({'success': True, 'geofeatures': item.geofeatures, 'log': item.log, 'msg': None, 'original_filename': item.original_filename}, status=200)
return JsonResponse({'success': True, 'geofeatures': [], 'log': [], 'msg': 'uploaded data still processing'}, status=200)
return JsonResponse({'success': True, 'processing': False, 'geofeatures': item.geofeatures, 'log': item.log, 'msg': None, 'original_filename': item.original_filename}, status=200)
return JsonResponse({'success': True, 'processing': True, 'geofeatures': [], 'log': [], 'msg': 'uploaded data still processing'}, status=200)
except ImportQueue.DoesNotExist:
return JsonResponse({'success': False, 'msg': 'ID does not exist', 'code': 404}, status=400)
@login_required_401
def fetch_import_waiting(request):
user_items = ImportQueue.objects.exclude(data__contains=[]).filter(user=request.user).values('id', 'geofeatures', 'original_filename', 'raw_kml_hash', 'data', 'log', 'timestamp')
user_items = ImportQueue.objects.exclude(data__contains='[]').filter(user=request.user).values('id', 'geofeatures', 'original_filename', 'raw_kml_hash', 'data', 'log', 'timestamp')
data = json.loads(json.dumps(list(user_items), cls=DjangoJSONEncoder))
lock_manager = DBLockManager()
for i, item in enumerate(data):
@ -97,7 +97,7 @@ def fetch_import_waiting(request):
@login_required_401
def fetch_import_history(request):
user_items = ImportQueue.objects.filter(geofeatures__contains=[], user=request.user).values('id', 'original_filename', 'timestamp')
user_items = ImportQueue.objects.filter(geofeatures__contains='[]', user=request.user).values('id', 'original_filename', 'timestamp')
data = json.loads(json.dumps(list(user_items), cls=DjangoJSONEncoder))
return JsonResponse({'data': data})

2
src/geo-backend/geo_lib/daemon/workers/importer.py
View File

@ -14,7 +14,7 @@ from geo_lib.logging.database import log_to_db, DatabaseLogLevel, DatabaseLogSou
from geo_lib.time import get_time_ms
from geo_lib.types.feature import geojson_to_geofeature
_SQL_GET_UNPROCESSED_ITEMS = "SELECT * FROM public.data_importqueue WHERE geofeatures = '{}'::jsonb ORDER BY id ASC"
_SQL_GET_UNPROCESSED_ITEMS = "SELECT * FROM public.data_importqueue WHERE geofeatures = '[]'::jsonb ORDER BY id ASC"
_SQL_INSERT_PROCESSED_ITEM = "UPDATE public.data_importqueue SET geofeatures = %s, log = %s WHERE id = %s"
_SQL_DELETE_ITEM = "DELETE FROM public.data_importqueue WHERE id = %s"

8
src/geo-frontend/src/components/Home.vue
View File

@ -1,5 +1,11 @@
<template>
<p>Home page</p>
<div class="prose">
<h1>Home page</h1>
</div>
<div>
<a href="/#/import">Import</a>
</div>
</template>

24
src/geo-frontend/src/components/import/ImportHome.vue
View File

@ -1,9 +1,16 @@
<template>
<div class="prose mb-10">
<h1 class="mb-1">Import Data</h1>
</div>
<div class="mb-10">
<div>
<a class="text-blue-500 hover:text-blue-700" href="/#/import/upload">Upload Files</a>
</div>
<div class="prose mt-10">
<h3>Ready to Import</h3>
</div>
<Importqueue/>
<div class="prose mt-10">
@ -12,13 +19,14 @@
<table class="mt-6 w-full border-collapse">
<thead>
<tr class="bg-gray-100">
<th class="px-4 py-2 text-left">File Name</th>
<th class="px-4 py-2 text-left w-[50%]">File Name</th>
<th class="px-4 py-2">Date/Time Imported</th>
<th class="px-4 py-2 w-[10%]"></th>
</tr>
</thead>
<tbody>
<tr v-for="(item, index) in history" :key="`history-${index}`" class="border-t">
<td class="px-4 py-2">
<td class="px-4 py-2 w-[50%]">
<a :href="`${IMPORT_HISTORY_URL()}/${item.id}`" class="text-blue-500 hover:text-blue-700">{{
item.original_filename
}}</a>
@ -26,6 +34,16 @@
<td class="px-4 py-2 text-center">
{{ item.timestamp }}
</td>
<td class="px-4 py-2 w-[10%]">
</td>
</tr>
<tr v-if="historyIsLoading" class="animate-pulse border-t">
<td class="px-4 py-2 text-left w-[50%]">
<div class="w-32 h-8 bg-gray-200 rounded-s"></div>
</td>
<td class="px-4 py-2 text-center">
<div class="w-32 h-8 bg-gray-200 rounded-s mx-auto"></div>
</td>
</tr>
</tbody>
</table>
@ -48,6 +66,7 @@ export default {
data() {
return {
history: [],
historyIsLoading: true,
}
},
methods: {
@ -57,6 +76,7 @@ export default {
async fetchHistory() {
const response = await axios.get(IMPORT_HISTORY_URL)
this.history = response.data.data
this.historyIsLoading = false
},
},
async created() {

72
src/geo-frontend/src/components/import/ImportProcess.vue
View File

@ -1,9 +1,9 @@
<template>
<div class="prose mb-10">
<h1 class="mb-1">Process Import</h1>
<h2 class="mt-0">{{ originalFilename }}</h2>
<h2 v-if="originalFilename != null" class="mt-0">{{ originalFilename }}</h2>
<h2 v-else class="mt-0 invisible">loading...</h2>
</div>
<div v-if="msg !== '' && msg != null">
<div class="bg-red-500 p-4 rounded">
<p class="font-bold text-white">{{ msg }}</p>
@ -11,20 +11,21 @@
</div>
<!-- TODO: loading indicator -->
<div v-if="originalFilename != null" id="importLog"
<div id="importLog"
class="w-full my-10 mx-auto overflow-auto h-32 bg-white shadow rounded-lg p-4">
<h2 class="text-lg font-semibold text-gray-700 mb-2">Logs</h2>
<hr class="mb-4 border-t border-gray-200">
<ul class="space-y-2">
<li v-for="(item, index) in workerLog" :key="`item-${index}`" class="border-b border-gray-200 last:border-b-0">
<p class="text-sm font-bold text-gray-600">{{ item }}</p>
<p class="text-sm">{{ item }}</p>
</li>
</ul>
</div>
<Loader v-if="originalFilename == null"/>
<div>
<ul class="space-y-4">
<li v-for="(item, index) in itemsForUser" :key="`item-${index}`" class="bg-white shadow rounded-md p-4">
@ -109,15 +110,16 @@ import {GeoPoint, GeoLineString, GeoPolygon} from "@/assets/js/types/geofeature-
import {getCookie} from "@/assets/js/auth.js";
import flatPickr from 'vue-flatpickr-component';
import 'flatpickr/dist/flatpickr.css';
import Loader from "@/components/parts/Loader.vue";
// TODO: for each feature, query the DB and check if there is a duplicate. For points that's duplicate coords, for linestrings and polygons that's duplicate points
// TODO: auto-refresh if still processing
// TODO: redo the entire log feature to include local timestamps
export default {
computed: {
...mapState(["userInfo"]),
},
components: {Importqueue, flatPickr},
components: {Loader, Importqueue, flatPickr},
data() {
return {
msg: "",
@ -130,7 +132,6 @@ export default {
enableTime: true,
time_24hr: true,
dateFormat: 'Y-m-d H:i',
// timezone: 'UTC',
},
}
},
@ -189,7 +190,7 @@ export default {
}
}).then(response => {
if (response.data.success) {
this.msg = 'Changes saved successfully';
this.msg = 'Changes saved successfully.';
window.alert(this.msg);
} else {
this.msg = 'Error saving changes: ' + response.data.msg;
@ -203,35 +204,44 @@ export default {
}
,
beforeRouteEnter(to, from, next) {
let ready = false
next(async vm => {
if (vm.currentId !== vm.id) {
vm.msg = ""
vm.messages = []
vm.itemsForUser = []
vm.originalItems = []
vm.currentId = null
axios.get('/api/data/item/import/get/' + vm.id).then(response => {
if (!response.data.success) {
vm.handleError(response.data.msg)
} else {
vm.currentId = vm.id
if (Object.keys(response.data).length > 0) {
vm.originalFilename = response.data.original_filename
response.data.geofeatures.forEach((item) => {
vm.itemsForUser.push(vm.parseGeoJson(item))
})
vm.originalItems = JSON.parse(JSON.stringify(vm.itemsForUser))
while (!ready) {
vm.itemsForUser = []
vm.originalItems = []
vm.currentId = null
try {
const response = await axios.get('/api/data/item/import/get/' + vm.id)
if (!response.data.success) {
vm.handleError(response.data.msg)
} else {
vm.currentId = vm.id
if (Object.keys(response.data).length > 0) {
vm.originalFilename = response.data.original_filename
response.data.geofeatures.forEach((item) => {
vm.itemsForUser.push(vm.parseGeoJson(item))
})
vm.originalItems = JSON.parse(JSON.stringify(vm.itemsForUser))
}
if (!response.data.processing) {
vm.workerLog.push(response.data.msg)
vm.workerLog.concat(response.data.log)
ready = true
} else {
vm.workerLog = [`${new Date().toISOString()} -- uploaded data still processing`]
await new Promise(r => setTimeout(r, 1000));
}
}
vm.msg = response.data.msg
vm.workerLog = response.data.log
} catch (error) {
vm.handleError(error.message)
}
}).catch(error => {
vm.handleError(error.message)
})
}
}
})
}
,
},
}
</script>

54
src/geo-frontend/src/components/import/ImportUpload.vue
View File

@ -1,8 +1,11 @@
<template>
<div class="prose mb-10">
<h1 class="mb-1">Upload Data</h1>
</div>
<div class="mb-10">
<p class="text-lg font-semibold mb-2">Import Data</p>
<p class="text-gray-600 mb-2">Only KML/KMZ files supported.</p>
<p class="text-gray-600">
<p class="mb-2">Only KML/KMZ files supported.</p>
<p class="">
Be careful not to upload duplicate files of the opposite type. For example, do not upload both
<kbd class="bg-gray-200 text-gray-800 px-2 py-1 rounded">example.kml</kbd>
and <kbd class="bg-gray-200 text-gray-800 px-2 py-1 rounded">example.kmz</kbd>. Currently, the system can't detect
@ -12,17 +15,36 @@
<div class="relative w-[90%] mx-auto">
<div class="flex items-center">
<input id="uploadInput" :disabled="disableUpload" class="mr-4 px-4 py-2 border border-gray-300 rounded" type="file"
<input id="uploadInput" :disabled="disableUpload" class="mr-4 px-4 py-2 border border-gray-300 rounded"
type="file"
@change="onFileChange">
<button :disabled="disableUpload" class="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600 disabled:bg-gray-400 disabled:cursor-not-allowed"
<button :disabled="disableUpload"
class="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600 disabled:bg-gray-400 disabled:cursor-not-allowed"
@click="upload">
Upload
</button>
</div>
<div :class="{invisible: uploadProgress <= 0}" class="mt-4">
<div class="w-full bg-gray-200 rounded-full h-2.5">
<div :style="{ width: uploadProgress + '%' }" class="bg-blue-600 h-2.5 rounded-full"></div>
</div>
<div class="text-center mt-2">{{ uploadProgress }}%</div>
</div>
<div class="prose" v-html="uploadResponse"></div>
<div v-if="uploadMsg !== ''" class="mt-10 max-h-40 overflow-y-auto bg-gray-200 rounded-s p-5">
<strong>Message from Server:</strong><br>
{{ uploadMsg }}
</div>
</div>
<div v-if="uploadMsg !== ''" class="w-[90%] mx-auto mt-10" v-html="uploadMsg"></div>
<div class="prose mt-10">
<h3 class="inline">Ready to Import</h3>
<span v-if="loadingQueueList" class="italic mr-3">
Loading...
</span>
</div>
<Importqueue/>
</template>
@ -48,23 +70,29 @@ export default {
file: null,
disableUpload: false,
uploadMsg: "",
uploadProgress: 0,
loadingQueueList: false,
uploadResponse: ""
}
},
methods: {
async fetchQueueList() {
this.loadingQueueList = true
const response = await axios.get(IMPORT_QUEUE_LIST_URL)
const ourImportQueue = response.data.data.map((item) => new ImportQueueItem(item))
this.$store.commit('importQueue', ourImportQueue)
this.loadingQueueList = false
},
onFileChange(e) {
this.file = e.target.files[0]
const fileType = this.file.name.split('.').pop().toLowerCase()
if (fileType !== 'kmz' && fileType !== 'kml') {
alert('Invalid file type. Only KMZ and KML files are allowed.') // TODO: have this be a message on the page?
alert('Invalid file type. Only KMZ and KML files are allowed.')
e.target.value = "" // Reset the input value
}
},
async upload() {
this.uploadProgress = 0
this.uploadMsg = ""
if (this.file == null) {
return
@ -77,16 +105,20 @@ export default {
headers: {
'Content-Type': 'multipart/form-data',
'X-CSRFToken': this.userInfo.csrftoken
}
},
onUploadProgress: (progressEvent) => { // Add this block
this.uploadProgress = Math.round((progressEvent.loaded * 100) / progressEvent.total)
},
})
this.uploadMsg = `<p>${capitalizeFirstLetter(response.data.msg).trim(".")}.</p><p><a href="/#/import/process/${response.data.id}">Continue to Import</a>`
this.uploadMsg = capitalizeFirstLetter(response.data.msg).trim(".") + "."
this.uploadResponse = `<a href="/#/import/process/${response.data.id}">Continue to Import</a>`
await this.fetchQueueList()
this.file = null
document.getElementById("uploadInput").value = ""
this.disableUpload = false
} catch (error) {
this.handleError(error)
}
this.disableUpload = false
},
handleError(error) {
console.error("Upload failed:", error)

36
src/geo-frontend/src/components/import/parts/importqueue.vue
View File

@ -1,32 +1,45 @@
<template>
<div class="mt-4">
<button class="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600" @click="fetchQueueList">Refresh</button>
</div>
<!-- <div class="mt-4">-->
<!-- <button class="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600" @click="fetchQueueList">Refresh</button>-->
<!-- </div>-->
<table class="mt-6 w-full border-collapse">
<thead>
<tr class="bg-gray-100">
<th class="px-4 py-2 text-left">File Name</th>
<th class="px-4 py-2 text-left">Features</th>
<th class="px-4 py-2"></th>
<th class="px-4 py-2 text-left w-[50%]">File Name</th>
<th class="px-4 py-2 text-center">Features</th>
<th class="px-4 py-2 w-[10%]"></th>
</tr>
</thead>
<tbody>
<tr v-for="(item, index) in importQueue" :key="`item-${index}`" class="border-t">
<td class="px-4 py-2">
<td class="px-4 py-2 w-[50%]">
<a :href="`/#/import/process/${item.id}`" class="text-blue-500 hover:text-blue-700">{{
item.original_filename
}}</a>
</td>
<td class="px-4 py-2">
<td class="px-4 py-2 text-center">
{{ item.processing === true ? "processing" : item.feature_count }}
</td>
<td class="px-4 py-2">
<td class="px-4 py-2 w-[10%]">
<button class="px-4 py-2 bg-red-500 text-white rounded hover:bg-red-600" @click="deleteItem(item, index)">
Delete
</button>
</td>
</tr>
<tr v-if="isLoading && importQueue.length === 0" class="animate-pulse border-t">
<td class="px-4 py-2 text-left w-[50%]">
<div class="w-32 h-8 bg-gray-200 rounded-s"></div>
</td>
<td class="px-4 py-2 text-center">
<div class="w-32 h-8 bg-gray-200 rounded-s mx-auto"></div>
</td>
<td class="px-4 py-2 invisible w-[10%]">
<button class="px-4 py-2 bg-red-500 text-white rounded hover:bg-red-600">
Delete
</button>
</td>
</tr>
</tbody>
</table>
</template>
@ -45,13 +58,16 @@ export default {
components: {},
mixins: [authMixin],
data() {
return {}
return {
isLoading: true,
}
},
methods: {
async fetchQueueList() {
const response = await axios.get(IMPORT_QUEUE_LIST_URL)
const ourImportQueue = response.data.data.map((item) => new ImportQueueItem(item))
this.$store.commit('importQueue', ourImportQueue)
this.isLoading = false
},
async deleteItem(item, index) {
if (window.confirm(`Delete "${item.original_filename}" (#${item.id})`))

22
src/geo-frontend/src/components/parts/Loader.vue Normal file
View File

@ -0,0 +1,22 @@
<template>
<div role="status">
<svg aria-hidden="true" class="h-8 text-gray-200 animate-spin dark:text-gray-600 fill-blue-600 w-full"
fill="none" viewBox="0 0 100 101" xmlns="http://www.w3.org/2000/svg">
<path
d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z"
fill="currentColor"/>
<path
d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z"
fill="currentFill"/>
</svg>
<span class="sr-only">Loading...</span>
</div>
</template>
<script setup>
</script>
<style scoped>
</style>