continue styling the import

This commit is contained in:
Cyberes 2024-09-12 14:53:40 -06:00
parent fcb13f4842
commit 4be332aff6
8 changed files with 167 additions and 61 deletions

View File

@ -74,17 +74,17 @@ def fetch_import_queue(request, item_id):
try: try:
item = ImportQueue.objects.get(id=item_id) item = ImportQueue.objects.get(id=item_id)
if item.user_id != request.user.id: if item.user_id != request.user.id:
return JsonResponse({'success': False, 'msg': 'not authorized to view this item', 'code': 403}, status=400) return JsonResponse({'success': False, 'processing': False, 'msg': 'not authorized to view this item', 'code': 403}, status=400)
if not lock_manager.is_locked('data_importqueue', item.id) and (len(item.geofeatures) or len(item.log)): if not lock_manager.is_locked('data_importqueue', item.id) and (len(item.geofeatures) or len(item.log)):
return JsonResponse({'success': True, 'geofeatures': item.geofeatures, 'log': item.log, 'msg': None, 'original_filename': item.original_filename}, status=200) return JsonResponse({'success': True, 'processing': False, 'geofeatures': item.geofeatures, 'log': item.log, 'msg': None, 'original_filename': item.original_filename}, status=200)
return JsonResponse({'success': True, 'geofeatures': [], 'log': [], 'msg': 'uploaded data still processing'}, status=200) return JsonResponse({'success': True, 'processing': True, 'geofeatures': [], 'log': [], 'msg': 'uploaded data still processing'}, status=200)
except ImportQueue.DoesNotExist: except ImportQueue.DoesNotExist:
return JsonResponse({'success': False, 'msg': 'ID does not exist', 'code': 404}, status=400) return JsonResponse({'success': False, 'msg': 'ID does not exist', 'code': 404}, status=400)
@login_required_401 @login_required_401
def fetch_import_waiting(request): def fetch_import_waiting(request):
user_items = ImportQueue.objects.exclude(data__contains=[]).filter(user=request.user).values('id', 'geofeatures', 'original_filename', 'raw_kml_hash', 'data', 'log', 'timestamp') user_items = ImportQueue.objects.exclude(data__contains='[]').filter(user=request.user).values('id', 'geofeatures', 'original_filename', 'raw_kml_hash', 'data', 'log', 'timestamp')
data = json.loads(json.dumps(list(user_items), cls=DjangoJSONEncoder)) data = json.loads(json.dumps(list(user_items), cls=DjangoJSONEncoder))
lock_manager = DBLockManager() lock_manager = DBLockManager()
for i, item in enumerate(data): for i, item in enumerate(data):
@ -97,7 +97,7 @@ def fetch_import_waiting(request):
@login_required_401 @login_required_401
def fetch_import_history(request): def fetch_import_history(request):
user_items = ImportQueue.objects.filter(geofeatures__contains=[], user=request.user).values('id', 'original_filename', 'timestamp') user_items = ImportQueue.objects.filter(geofeatures__contains='[]', user=request.user).values('id', 'original_filename', 'timestamp')
data = json.loads(json.dumps(list(user_items), cls=DjangoJSONEncoder)) data = json.loads(json.dumps(list(user_items), cls=DjangoJSONEncoder))
return JsonResponse({'data': data}) return JsonResponse({'data': data})

View File

@ -14,7 +14,7 @@ from geo_lib.logging.database import log_to_db, DatabaseLogLevel, DatabaseLogSou
from geo_lib.time import get_time_ms from geo_lib.time import get_time_ms
from geo_lib.types.feature import geojson_to_geofeature from geo_lib.types.feature import geojson_to_geofeature
_SQL_GET_UNPROCESSED_ITEMS = "SELECT * FROM public.data_importqueue WHERE geofeatures = '{}'::jsonb ORDER BY id ASC" _SQL_GET_UNPROCESSED_ITEMS = "SELECT * FROM public.data_importqueue WHERE geofeatures = '[]'::jsonb ORDER BY id ASC"
_SQL_INSERT_PROCESSED_ITEM = "UPDATE public.data_importqueue SET geofeatures = %s, log = %s WHERE id = %s" _SQL_INSERT_PROCESSED_ITEM = "UPDATE public.data_importqueue SET geofeatures = %s, log = %s WHERE id = %s"
_SQL_DELETE_ITEM = "DELETE FROM public.data_importqueue WHERE id = %s" _SQL_DELETE_ITEM = "DELETE FROM public.data_importqueue WHERE id = %s"

View File

@ -1,5 +1,11 @@
<template> <template>
<p>Home page</p> <div class="prose">
<h1>Home page</h1>
</div>
<div>
<a href="/#/import">Import</a>
</div>
</template> </template>

View File

@ -1,9 +1,16 @@
<template> <template>
<div class="prose mb-10">
<h1 class="mb-1">Import Data</h1>
</div>
<div class="mb-10"> <div class="mb-10">
<div> <div>
<a class="text-blue-500 hover:text-blue-700" href="/#/import/upload">Upload Files</a> <a class="text-blue-500 hover:text-blue-700" href="/#/import/upload">Upload Files</a>
</div> </div>
<div class="prose mt-10">
<h3>Ready to Import</h3>
</div>
<Importqueue/> <Importqueue/>
<div class="prose mt-10"> <div class="prose mt-10">
@ -12,13 +19,14 @@
<table class="mt-6 w-full border-collapse"> <table class="mt-6 w-full border-collapse">
<thead> <thead>
<tr class="bg-gray-100"> <tr class="bg-gray-100">
<th class="px-4 py-2 text-left">File Name</th> <th class="px-4 py-2 text-left w-[50%]">File Name</th>
<th class="px-4 py-2">Date/Time Imported</th> <th class="px-4 py-2">Date/Time Imported</th>
<th class="px-4 py-2 w-[10%]"></th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
<tr v-for="(item, index) in history" :key="`history-${index}`" class="border-t"> <tr v-for="(item, index) in history" :key="`history-${index}`" class="border-t">
<td class="px-4 py-2"> <td class="px-4 py-2 w-[50%]">
<a :href="`${IMPORT_HISTORY_URL()}/${item.id}`" class="text-blue-500 hover:text-blue-700">{{ <a :href="`${IMPORT_HISTORY_URL()}/${item.id}`" class="text-blue-500 hover:text-blue-700">{{
item.original_filename item.original_filename
}}</a> }}</a>
@ -26,6 +34,16 @@
<td class="px-4 py-2 text-center"> <td class="px-4 py-2 text-center">
{{ item.timestamp }} {{ item.timestamp }}
</td> </td>
<td class="px-4 py-2 w-[10%]">
</td>
</tr>
<tr v-if="historyIsLoading" class="animate-pulse border-t">
<td class="px-4 py-2 text-left w-[50%]">
<div class="w-32 h-8 bg-gray-200 rounded-s"></div>
</td>
<td class="px-4 py-2 text-center">
<div class="w-32 h-8 bg-gray-200 rounded-s mx-auto"></div>
</td>
</tr> </tr>
</tbody> </tbody>
</table> </table>
@ -48,6 +66,7 @@ export default {
data() { data() {
return { return {
history: [], history: [],
historyIsLoading: true,
} }
}, },
methods: { methods: {
@ -57,6 +76,7 @@ export default {
async fetchHistory() { async fetchHistory() {
const response = await axios.get(IMPORT_HISTORY_URL) const response = await axios.get(IMPORT_HISTORY_URL)
this.history = response.data.data this.history = response.data.data
this.historyIsLoading = false
}, },
}, },
async created() { async created() {

View File

@ -1,9 +1,9 @@
<template> <template>
<div class="prose mb-10"> <div class="prose mb-10">
<h1 class="mb-1">Process Import</h1> <h1 class="mb-1">Process Import</h1>
<h2 class="mt-0">{{ originalFilename }}</h2> <h2 v-if="originalFilename != null" class="mt-0">{{ originalFilename }}</h2>
<h2 v-else class="mt-0 invisible">loading...</h2>
</div> </div>
<div v-if="msg !== '' && msg != null"> <div v-if="msg !== '' && msg != null">
<div class="bg-red-500 p-4 rounded"> <div class="bg-red-500 p-4 rounded">
<p class="font-bold text-white">{{ msg }}</p> <p class="font-bold text-white">{{ msg }}</p>
@ -11,20 +11,21 @@
</div> </div>
<!-- TODO: loading indicator --> <div id="importLog"
<div v-if="originalFilename != null" id="importLog"
class="w-full my-10 mx-auto overflow-auto h-32 bg-white shadow rounded-lg p-4"> class="w-full my-10 mx-auto overflow-auto h-32 bg-white shadow rounded-lg p-4">
<h2 class="text-lg font-semibold text-gray-700 mb-2">Logs</h2> <h2 class="text-lg font-semibold text-gray-700 mb-2">Logs</h2>
<hr class="mb-4 border-t border-gray-200"> <hr class="mb-4 border-t border-gray-200">
<ul class="space-y-2"> <ul class="space-y-2">
<li v-for="(item, index) in workerLog" :key="`item-${index}`" class="border-b border-gray-200 last:border-b-0"> <li v-for="(item, index) in workerLog" :key="`item-${index}`" class="border-b border-gray-200 last:border-b-0">
<p class="text-sm font-bold text-gray-600">{{ item }}</p> <p class="text-sm">{{ item }}</p>
</li> </li>
</ul> </ul>
</div> </div>
<Loader v-if="originalFilename == null"/>
<div> <div>
<ul class="space-y-4"> <ul class="space-y-4">
<li v-for="(item, index) in itemsForUser" :key="`item-${index}`" class="bg-white shadow rounded-md p-4"> <li v-for="(item, index) in itemsForUser" :key="`item-${index}`" class="bg-white shadow rounded-md p-4">
@ -109,15 +110,16 @@ import {GeoPoint, GeoLineString, GeoPolygon} from "@/assets/js/types/geofeature-
import {getCookie} from "@/assets/js/auth.js"; import {getCookie} from "@/assets/js/auth.js";
import flatPickr from 'vue-flatpickr-component'; import flatPickr from 'vue-flatpickr-component';
import 'flatpickr/dist/flatpickr.css'; import 'flatpickr/dist/flatpickr.css';
import Loader from "@/components/parts/Loader.vue";
// TODO: for each feature, query the DB and check if there is a duplicate. For points that's duplicate coords, for linestrings and polygons that's duplicate points // TODO: for each feature, query the DB and check if there is a duplicate. For points that's duplicate coords, for linestrings and polygons that's duplicate points
// TODO: auto-refresh if still processing // TODO: redo the entire log feature to include local timestamps
export default { export default {
computed: { computed: {
...mapState(["userInfo"]), ...mapState(["userInfo"]),
}, },
components: {Importqueue, flatPickr}, components: {Loader, Importqueue, flatPickr},
data() { data() {
return { return {
msg: "", msg: "",
@ -130,7 +132,6 @@ export default {
enableTime: true, enableTime: true,
time_24hr: true, time_24hr: true,
dateFormat: 'Y-m-d H:i', dateFormat: 'Y-m-d H:i',
// timezone: 'UTC',
}, },
} }
}, },
@ -189,7 +190,7 @@ export default {
} }
}).then(response => { }).then(response => {
if (response.data.success) { if (response.data.success) {
this.msg = 'Changes saved successfully'; this.msg = 'Changes saved successfully.';
window.alert(this.msg); window.alert(this.msg);
} else { } else {
this.msg = 'Error saving changes: ' + response.data.msg; this.msg = 'Error saving changes: ' + response.data.msg;
@ -203,35 +204,44 @@ export default {
} }
, ,
beforeRouteEnter(to, from, next) { beforeRouteEnter(to, from, next) {
let ready = false
next(async vm => { next(async vm => {
if (vm.currentId !== vm.id) { if (vm.currentId !== vm.id) {
vm.msg = "" vm.msg = ""
vm.messages = [] vm.messages = []
vm.itemsForUser = [] while (!ready) {
vm.originalItems = [] vm.itemsForUser = []
vm.currentId = null vm.originalItems = []
axios.get('/api/data/item/import/get/' + vm.id).then(response => { vm.currentId = null
if (!response.data.success) { try {
vm.handleError(response.data.msg) const response = await axios.get('/api/data/item/import/get/' + vm.id)
} else { if (!response.data.success) {
vm.currentId = vm.id vm.handleError(response.data.msg)
if (Object.keys(response.data).length > 0) { } else {
vm.originalFilename = response.data.original_filename vm.currentId = vm.id
response.data.geofeatures.forEach((item) => { if (Object.keys(response.data).length > 0) {
vm.itemsForUser.push(vm.parseGeoJson(item)) vm.originalFilename = response.data.original_filename
}) response.data.geofeatures.forEach((item) => {
vm.originalItems = JSON.parse(JSON.stringify(vm.itemsForUser)) vm.itemsForUser.push(vm.parseGeoJson(item))
})
vm.originalItems = JSON.parse(JSON.stringify(vm.itemsForUser))
}
if (!response.data.processing) {
vm.workerLog.push(response.data.msg)
vm.workerLog.concat(response.data.log)
ready = true
} else {
vm.workerLog = [`${new Date().toISOString()} -- uploaded data still processing`]
await new Promise(r => setTimeout(r, 1000));
}
} }
vm.msg = response.data.msg } catch (error) {
vm.workerLog = response.data.log vm.handleError(error.message)
} }
}).catch(error => { }
vm.handleError(error.message)
})
} }
}) })
} },
,
} }
</script> </script>

View File

@ -1,8 +1,11 @@
<template> <template>
<div class="prose mb-10">
<h1 class="mb-1">Upload Data</h1>
</div>
<div class="mb-10"> <div class="mb-10">
<p class="text-lg font-semibold mb-2">Import Data</p> <p class="mb-2">Only KML/KMZ files supported.</p>
<p class="text-gray-600 mb-2">Only KML/KMZ files supported.</p> <p class="">
<p class="text-gray-600">
Be careful not to upload duplicate files of the opposite type. For example, do not upload both Be careful not to upload duplicate files of the opposite type. For example, do not upload both
<kbd class="bg-gray-200 text-gray-800 px-2 py-1 rounded">example.kml</kbd> <kbd class="bg-gray-200 text-gray-800 px-2 py-1 rounded">example.kml</kbd>
and <kbd class="bg-gray-200 text-gray-800 px-2 py-1 rounded">example.kmz</kbd>. Currently, the system can't detect and <kbd class="bg-gray-200 text-gray-800 px-2 py-1 rounded">example.kmz</kbd>. Currently, the system can't detect
@ -12,17 +15,36 @@
<div class="relative w-[90%] mx-auto"> <div class="relative w-[90%] mx-auto">
<div class="flex items-center"> <div class="flex items-center">
<input id="uploadInput" :disabled="disableUpload" class="mr-4 px-4 py-2 border border-gray-300 rounded" type="file" <input id="uploadInput" :disabled="disableUpload" class="mr-4 px-4 py-2 border border-gray-300 rounded"
type="file"
@change="onFileChange"> @change="onFileChange">
<button :disabled="disableUpload" class="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600 disabled:bg-gray-400 disabled:cursor-not-allowed" <button :disabled="disableUpload"
class="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600 disabled:bg-gray-400 disabled:cursor-not-allowed"
@click="upload"> @click="upload">
Upload Upload
</button> </button>
</div> </div>
<div :class="{invisible: uploadProgress <= 0}" class="mt-4">
<div class="w-full bg-gray-200 rounded-full h-2.5">
<div :style="{ width: uploadProgress + '%' }" class="bg-blue-600 h-2.5 rounded-full"></div>
</div>
<div class="text-center mt-2">{{ uploadProgress }}%</div>
</div>
<div class="prose" v-html="uploadResponse"></div>
<div v-if="uploadMsg !== ''" class="mt-10 max-h-40 overflow-y-auto bg-gray-200 rounded-s p-5">
<strong>Message from Server:</strong><br>
{{ uploadMsg }}
</div>
</div> </div>
<div v-if="uploadMsg !== ''" class="w-[90%] mx-auto mt-10" v-html="uploadMsg"></div> <div class="prose mt-10">
<h3 class="inline">Ready to Import</h3>
<span v-if="loadingQueueList" class="italic mr-3">
Loading...
</span>
</div>
<Importqueue/> <Importqueue/>
</template> </template>
@ -48,23 +70,29 @@ export default {
file: null, file: null,
disableUpload: false, disableUpload: false,
uploadMsg: "", uploadMsg: "",
uploadProgress: 0,
loadingQueueList: false,
uploadResponse: ""
} }
}, },
methods: { methods: {
async fetchQueueList() { async fetchQueueList() {
this.loadingQueueList = true
const response = await axios.get(IMPORT_QUEUE_LIST_URL) const response = await axios.get(IMPORT_QUEUE_LIST_URL)
const ourImportQueue = response.data.data.map((item) => new ImportQueueItem(item)) const ourImportQueue = response.data.data.map((item) => new ImportQueueItem(item))
this.$store.commit('importQueue', ourImportQueue) this.$store.commit('importQueue', ourImportQueue)
this.loadingQueueList = false
}, },
onFileChange(e) { onFileChange(e) {
this.file = e.target.files[0] this.file = e.target.files[0]
const fileType = this.file.name.split('.').pop().toLowerCase() const fileType = this.file.name.split('.').pop().toLowerCase()
if (fileType !== 'kmz' && fileType !== 'kml') { if (fileType !== 'kmz' && fileType !== 'kml') {
alert('Invalid file type. Only KMZ and KML files are allowed.') // TODO: have this be a message on the page? alert('Invalid file type. Only KMZ and KML files are allowed.')
e.target.value = "" // Reset the input value e.target.value = "" // Reset the input value
} }
}, },
async upload() { async upload() {
this.uploadProgress = 0
this.uploadMsg = "" this.uploadMsg = ""
if (this.file == null) { if (this.file == null) {
return return
@ -77,16 +105,20 @@ export default {
headers: { headers: {
'Content-Type': 'multipart/form-data', 'Content-Type': 'multipart/form-data',
'X-CSRFToken': this.userInfo.csrftoken 'X-CSRFToken': this.userInfo.csrftoken
} },
onUploadProgress: (progressEvent) => { // Add this block
this.uploadProgress = Math.round((progressEvent.loaded * 100) / progressEvent.total)
},
}) })
this.uploadMsg = `<p>${capitalizeFirstLetter(response.data.msg).trim(".")}.</p><p><a href="/#/import/process/${response.data.id}">Continue to Import</a>` this.uploadMsg = capitalizeFirstLetter(response.data.msg).trim(".") + "."
this.uploadResponse = `<a href="/#/import/process/${response.data.id}">Continue to Import</a>`
await this.fetchQueueList() await this.fetchQueueList()
this.file = null this.file = null
document.getElementById("uploadInput").value = "" document.getElementById("uploadInput").value = ""
this.disableUpload = false
} catch (error) { } catch (error) {
this.handleError(error) this.handleError(error)
} }
this.disableUpload = false
}, },
handleError(error) { handleError(error) {
console.error("Upload failed:", error) console.error("Upload failed:", error)

View File

@ -1,32 +1,45 @@
<template> <template>
<div class="mt-4"> <!-- <div class="mt-4">-->
<button class="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600" @click="fetchQueueList">Refresh</button> <!-- <button class="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600" @click="fetchQueueList">Refresh</button>-->
</div> <!-- </div>-->
<table class="mt-6 w-full border-collapse"> <table class="mt-6 w-full border-collapse">
<thead> <thead>
<tr class="bg-gray-100"> <tr class="bg-gray-100">
<th class="px-4 py-2 text-left">File Name</th> <th class="px-4 py-2 text-left w-[50%]">File Name</th>
<th class="px-4 py-2 text-left">Features</th> <th class="px-4 py-2 text-center">Features</th>
<th class="px-4 py-2"></th> <th class="px-4 py-2 w-[10%]"></th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
<tr v-for="(item, index) in importQueue" :key="`item-${index}`" class="border-t"> <tr v-for="(item, index) in importQueue" :key="`item-${index}`" class="border-t">
<td class="px-4 py-2"> <td class="px-4 py-2 w-[50%]">
<a :href="`/#/import/process/${item.id}`" class="text-blue-500 hover:text-blue-700">{{ <a :href="`/#/import/process/${item.id}`" class="text-blue-500 hover:text-blue-700">{{
item.original_filename item.original_filename
}}</a> }}</a>
</td> </td>
<td class="px-4 py-2"> <td class="px-4 py-2 text-center">
{{ item.processing === true ? "processing" : item.feature_count }} {{ item.processing === true ? "processing" : item.feature_count }}
</td> </td>
<td class="px-4 py-2"> <td class="px-4 py-2 w-[10%]">
<button class="px-4 py-2 bg-red-500 text-white rounded hover:bg-red-600" @click="deleteItem(item, index)"> <button class="px-4 py-2 bg-red-500 text-white rounded hover:bg-red-600" @click="deleteItem(item, index)">
Delete Delete
</button> </button>
</td> </td>
</tr> </tr>
<tr v-if="isLoading && importQueue.length === 0" class="animate-pulse border-t">
<td class="px-4 py-2 text-left w-[50%]">
<div class="w-32 h-8 bg-gray-200 rounded-s"></div>
</td>
<td class="px-4 py-2 text-center">
<div class="w-32 h-8 bg-gray-200 rounded-s mx-auto"></div>
</td>
<td class="px-4 py-2 invisible w-[10%]">
<button class="px-4 py-2 bg-red-500 text-white rounded hover:bg-red-600">
Delete
</button>
</td>
</tr>
</tbody> </tbody>
</table> </table>
</template> </template>
@ -45,13 +58,16 @@ export default {
components: {}, components: {},
mixins: [authMixin], mixins: [authMixin],
data() { data() {
return {} return {
isLoading: true,
}
}, },
methods: { methods: {
async fetchQueueList() { async fetchQueueList() {
const response = await axios.get(IMPORT_QUEUE_LIST_URL) const response = await axios.get(IMPORT_QUEUE_LIST_URL)
const ourImportQueue = response.data.data.map((item) => new ImportQueueItem(item)) const ourImportQueue = response.data.data.map((item) => new ImportQueueItem(item))
this.$store.commit('importQueue', ourImportQueue) this.$store.commit('importQueue', ourImportQueue)
this.isLoading = false
}, },
async deleteItem(item, index) { async deleteItem(item, index) {
if (window.confirm(`Delete "${item.original_filename}" (#${item.id})`)) if (window.confirm(`Delete "${item.original_filename}" (#${item.id})`))

View File

@ -0,0 +1,22 @@
<template>
<div role="status">
<svg aria-hidden="true" class="h-8 text-gray-200 animate-spin dark:text-gray-600 fill-blue-600 w-full"
fill="none" viewBox="0 0 100 101" xmlns="http://www.w3.org/2000/svg">
<path
d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z"
fill="currentColor"/>
<path
d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z"
fill="currentFill"/>
</svg>
<span class="sr-only">Loading...</span>
</div>
</template>
<script setup>
</script>
<style scoped>
</style>