style more shit, improve import process

This commit is contained in:
Cyberes 2024-09-10 20:07:12 -06:00
parent 12d8b441c0
commit aa2a80a299
18 changed files with 218 additions and 100 deletions

View File

@ -1,11 +1,13 @@
from django.urls import path
from data.views.import_item import upload_item, fetch_import_queue, fetch_queued, delete_import_queue, update_imported_item
from data.views.import_item import upload_item, fetch_import_queue, fetch_import_waiting, delete_import_item, update_import_item, fetch_import_history, fetch_import_history_item
urlpatterns = [
path('item/import/upload/', upload_item, name='upload_file'),
path('item/import/get/<int:item_id>', fetch_import_queue, name='fetch_import_queue'),
path('item/import/get/mine', fetch_queued, name='fetch_queued'),
path('item/import/delete/<int:id>', delete_import_queue, name='delete_import_queue'),
path('item/import/update/<int:item_id>', update_imported_item),
path('item/import/upload', upload_item),
path('item/import/get/<int:item_id>', fetch_import_queue),
path('item/import/get', fetch_import_waiting),
path('item/import/get/history', fetch_import_history),
path('item/import/get/history/<int:item_id>', fetch_import_history_item),
path('item/import/delete/<int:id>', delete_import_item),
path('item/import/update/<int:item_id>', update_import_item),
]

View File

@ -72,19 +72,19 @@ def fetch_import_queue(request, item_id):
return JsonResponse({'success': False, 'msg': 'ID not provided', 'code': 400}, status=400)
lock_manager = DBLockManager()
try:
queue = ImportQueue.objects.get(id=item_id)
if queue.user_id != request.user.id:
item = ImportQueue.objects.get(id=item_id)
if item.user_id != request.user.id:
return JsonResponse({'success': False, 'msg': 'not authorized to view this item', 'code': 403}, status=400)
if not lock_manager.is_locked('data_importqueue', queue.id) and (len(queue.geofeatures) or len(queue.log)):
return JsonResponse({'success': True, 'geofeatures': queue.geofeatures, 'log': queue.log, 'msg': None}, status=200)
if not lock_manager.is_locked('data_importqueue', item.id) and (len(item.geofeatures) or len(item.log)):
return JsonResponse({'success': True, 'geofeatures': item.geofeatures, 'log': item.log, 'msg': None, 'original_filename': item.original_filename}, status=200)
return JsonResponse({'success': True, 'geofeatures': [], 'log': [], 'msg': 'uploaded data still processing'}, status=200)
except ImportQueue.DoesNotExist:
return JsonResponse({'success': False, 'msg': 'ID does not exist', 'code': 404}, status=400)
@login_required_401
def fetch_queued(request):
user_items = ImportQueue.objects.exclude(geofeatures__len=0).filter(user=request.user).values('id', 'geofeatures', 'original_filename', 'raw_kml_hash', 'data', 'log', 'timestamp')
def fetch_import_waiting(request):
user_items = ImportQueue.objects.exclude(data__contains=[]).filter(user=request.user).values('id', 'geofeatures', 'original_filename', 'raw_kml_hash', 'data', 'log', 'timestamp')
data = json.loads(json.dumps(list(user_items), cls=DjangoJSONEncoder))
lock_manager = DBLockManager()
for i, item in enumerate(data):
@ -96,7 +96,25 @@ def fetch_queued(request):
@login_required_401
def delete_import_queue(request, id):
def fetch_import_history(request):
user_items = ImportQueue.objects.filter(geofeatures__contains=[], user=request.user).values('id', 'original_filename', 'timestamp')
data = json.loads(json.dumps(list(user_items), cls=DjangoJSONEncoder))
return JsonResponse({'data': data})
@login_required_401
def fetch_import_history_item(request, item_id: int):
item = ImportQueue.objects.get(id=item_id)
if item.user_id != request.user.id:
return JsonResponse({'success': False, 'msg': 'not authorized to view this item', 'code': 403}, status=400)
response = HttpResponse(item.raw_kml, content_type='application/octet-stream')
response['Content-Disposition'] = 'attachment; filename="%s"' % item.original_filename
return response
@login_required_401
def delete_import_item(request, id):
if request.method == 'DELETE':
try:
queue = ImportQueue.objects.get(id=id)
@ -110,7 +128,7 @@ def delete_import_queue(request, id):
@login_required_401
@csrf_protect # TODO: put this on all routes
@require_http_methods(["PUT"])
def update_imported_item(request, item_id):
def update_import_item(request, item_id):
try:
queue = ImportQueue.objects.get(id=item_id)
except ImportQueue.DoesNotExist:

View File

@ -14,7 +14,7 @@ from geo_lib.logging.database import log_to_db, DatabaseLogLevel, DatabaseLogSou
from geo_lib.time import get_time_ms
from geo_lib.types.feature import geojson_to_geofeature
_SQL_GET_UNPROCESSED_ITEMS = "SELECT * FROM public.data_importqueue WHERE geofeatures = '[]' AND log = '[]' ORDER BY id ASC"
_SQL_GET_UNPROCESSED_ITEMS = "SELECT * FROM public.data_importqueue WHERE geofeatures = '{}'::jsonb ORDER BY id ASC"
_SQL_INSERT_PROCESSED_ITEM = "UPDATE public.data_importqueue SET geofeatures = %s, log = %s WHERE id = %s"
_SQL_DELETE_ITEM = "DELETE FROM public.data_importqueue WHERE id = %s"

View File

@ -0,0 +1,7 @@
1. Style main import page
2. Fix created field reset on edit imported
3. Implement refresh on edit imported
4. Style messages/log on edit imported
5. Implement upload working animation on edit imported
- For tracks, set the created date to the timestamp of the first point in the track

View File

@ -1,5 +1,5 @@
"""
ASGI config for account project.
ASGI config for website project.
It exposes the ASGI callable as a module-level variable named ``application``.
@ -11,6 +11,6 @@ import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'account.settings')
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'website.settings')
application = get_asgi_application()

View File

@ -1,5 +1,5 @@
"""
Django settings for account project.
Django settings for website project.
Generated by 'django-admin startproject' using Django 5.0.6.
@ -48,10 +48,10 @@ MIDDLEWARE = [
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'account.middleware.CustomHeaderMiddleware',
'website.middleware.CustomHeaderMiddleware',
]
ROOT_URLCONF = 'account.urls'
ROOT_URLCONF = 'website.urls'
TEMPLATES = [
{
@ -69,7 +69,7 @@ TEMPLATES = [
},
]
WSGI_APPLICATION = 'account.wsgi.application'
WSGI_APPLICATION = 'website.wsgi.application'
# Database
# https://docs.djangoproject.com/en/5.0/ref/settings/#databases

View File

@ -1,5 +1,5 @@
"""
URL configuration for account project.
URL configuration for website project.
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/5.0/topics/http/urls/
@ -18,11 +18,11 @@ from django.conf.urls import include
from django.contrib import admin
from django.urls import path, re_path
from account.views import index
from website.views import index
urlpatterns = [
path('', index),
re_path(r"^account/", include("django.contrib.auth.urls")),
re_path(r"^website/", include("django.contrib.auth.urls")),
path('admin/', admin.site.urls),
path('', include("users.urls")),
path('api/data/', include("data.urls"))

View File

@ -1,5 +1,7 @@
from django.contrib.auth.decorators import login_required
from django.shortcuts import render
@login_required
def index(request):
return render(request, "index.html")

View File

@ -1,5 +1,5 @@
"""
WSGI config for account project.
WSGI config for website project.
It exposes the WSGI callable as a module-level variable named ``application``.
@ -11,6 +11,6 @@ import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'account.settings')
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'website.settings')
application = get_wsgi_application()

View File

@ -19,6 +19,7 @@
"vuex": "^4.1.0"
},
"devDependencies": {
"@tailwindcss/typography": "^0.5.15",
"@vitejs/plugin-vue": "^5.0.4",
"autoprefixer": "^10.4.19",
"postcss": "^8.4.38",
@ -734,6 +735,36 @@
"win32"
]
},
"node_modules/@tailwindcss/typography": {
"version": "0.5.15",
"resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.15.tgz",
"integrity": "sha512-AqhlCXl+8grUz8uqExv5OTtgpjuVIwFTSXTrh8y9/pw6q2ek7fJ+Y8ZEVw7EB2DCcuCOtEjf9w3+J3rzts01uA==",
"dev": true,
"license": "MIT",
"dependencies": {
"lodash.castarray": "^4.4.0",
"lodash.isplainobject": "^4.0.6",
"lodash.merge": "^4.6.2",
"postcss-selector-parser": "6.0.10"
},
"peerDependencies": {
"tailwindcss": ">=3.0.0 || insiders || >=4.0.0-alpha.20"
}
},
"node_modules/@tailwindcss/typography/node_modules/postcss-selector-parser": {
"version": "6.0.10",
"resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz",
"integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==",
"dev": true,
"license": "MIT",
"dependencies": {
"cssesc": "^3.0.0",
"util-deprecate": "^1.0.2"
},
"engines": {
"node": ">=4"
}
},
"node_modules/@types/estree": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz",
@ -1588,6 +1619,27 @@
"integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==",
"dev": true
},
"node_modules/lodash.castarray": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.castarray/-/lodash.castarray-4.4.0.tgz",
"integrity": "sha512-aVx8ztPv7/2ULbArGJ2Y42bG1mEQ5mGjpdvrbJcJFU3TbYybe+QlLS4pst9zV52ymy2in1KpFPiZnAOATxD4+Q==",
"dev": true,
"license": "MIT"
},
"node_modules/lodash.isplainobject": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
"integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==",
"dev": true,
"license": "MIT"
},
"node_modules/lodash.merge": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
"integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
"dev": true,
"license": "MIT"
},
"node_modules/lru-cache": {
"version": "10.2.2",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz",

View File

@ -20,6 +20,7 @@
"vuex": "^4.1.0"
},
"devDependencies": {
"@tailwindcss/typography": "^0.5.15",
"@vitejs/plugin-vue": "^5.0.4",
"autoprefixer": "^10.4.19",
"postcss": "^8.4.38",

View File

@ -1 +1,2 @@
export const IMPORT_QUEUE_LIST_URL = "/api/data/item/import/get/mine"
export const IMPORT_QUEUE_LIST_URL = "/api/data/item/import/get"
export const IMPORT_HISTORY_URL = "/api/data/item/import/get/history"

View File

@ -1,10 +1,34 @@
<template>
<div class="mb-10">
<div>
<a href="/#/import/upload">Upload Files</a>
<a class="text-blue-500 hover:text-blue-700" href="/#/import/upload">Upload Files</a>
</div>
<Importqueue/>
<div class="prose mt-10">
<h3>Import History</h3>
</div>
<table class="mt-6 w-full border-collapse">
<thead>
<tr class="bg-gray-100">
<th class="px-4 py-2 text-left">File Name</th>
<th class="px-4 py-2">Date/Time Imported</th>
</tr>
</thead>
<tbody>
<tr v-for="(item, index) in history" :key="`history-${index}`" class="border-t">
<td class="px-4 py-2">
<a :href="`${IMPORT_HISTORY_URL()}/${item.id}`" class="text-blue-500 hover:text-blue-700">{{
item.original_filename
}}</a>
</td>
<td class="px-4 py-2 text-center">
{{ item.timestamp }}
</td>
</tr>
</tbody>
</table>
</div>
</template>
@ -12,8 +36,7 @@
import {mapState} from "vuex"
import {authMixin} from "@/assets/js/authMixin.js";
import axios from "axios";
import {IMPORT_QUEUE_LIST_URL} from "@/assets/js/import/url.js";
import {ImportQueueItem} from "@/assets/js/types/import-types"
import {IMPORT_HISTORY_URL} from "@/assets/js/import/url.js";
import Importqueue from "@/components/import/parts/importqueue.vue";
export default {
@ -23,36 +46,22 @@ export default {
components: {Importqueue},
mixins: [authMixin],
data() {
return {}
},
methods: {
async fetchQueueList() {
const response = await axios.get(IMPORT_QUEUE_LIST_URL)
const ourImportQueue = response.data.data.map((item) => new ImportQueueItem(item))
this.$store.commit('importQueue', ourImportQueue)
},
async deleteItem(item, index) {
if (window.confirm(`Delete "${item.original_filename}" (#${item.id})`))
try {
this.importQueue.splice(index, 1)
// TODO: add a message popup when delete is completed
const response = await axios.delete('/api/data/item/import/delete/' + item.id, {
headers: {
'X-CSRFToken': this.userInfo.csrftoken
}
})
if (!response.data.success) {
throw new Error("server reported failure")
}
await this.fetchQueueList()
} catch (error) {
alert(`Failed to delete ${item.id}: ${error.message}`)
this.importQueue.splice(index, 0, item)
}
return {
history: [],
}
},
// async created() {
// },
methods: {
IMPORT_HISTORY_URL() {
return IMPORT_HISTORY_URL
},
async fetchHistory() {
const response = await axios.get(IMPORT_HISTORY_URL)
this.history = response.data.data
},
},
async created() {
await this.fetchHistory()
},
// async mounted() {
// },
// beforeRouteEnter(to, from, next) {

View File

@ -1,20 +1,33 @@
<template>
<div v-if="msg !== ''">
<p class="font-bold">{{ msg }}</p>
<div class="prose mb-10">
<h1 class="mb-1">Process Import</h1>
<h2 class="mt-0">{{ originalFilename }}</h2>
</div>
<div v-if="msg !== '' && msg != null">
<div class="bg-red-500 p-4 rounded">
<p class="font-bold text-white">{{ msg }}</p>
</div>
</div>
<!-- TODO: loading indicator -->
<div id="importMessages">
<h2>Messages</h2>
<li v-for="(item, index) in workerLog" :key="`item-${index}`">
<p class="font-bold">{{ item }}</p>
</li>
<div v-if="originalFilename != null" id="importLog"
class="w-full my-10 mx-auto overflow-auto h-32 bg-white shadow rounded-lg p-4">
<h2 class="text-lg font-semibold text-gray-700 mb-2">Logs</h2>
<hr class="mb-4 border-t border-gray-200">
<ul class="space-y-2">
<li v-for="(item, index) in workerLog" :key="`item-${index}`" class="border-b border-gray-200 last:border-b-0">
<p class="text-sm font-bold text-gray-600">{{ item }}</p>
</li>
</ul>
</div>
<div>
<ul class="space-y-4">
<li v-for="(item, index) in itemsForUser" :key="`item-${index}`" class="bg-white shadow-md rounded-md p-4">
<li v-for="(item, index) in itemsForUser" :key="`item-${index}`" class="bg-white shadow rounded-md p-4">
<div class="mb-4">
<label class="block text-gray-700 font-bold mb-2">Name:</label>
<div class="flex items-center">
@ -35,7 +48,7 @@
</button>
</div>
</div>
<div class="">
<div>
<label class="block text-gray-700 font-bold mb-2">Created:</label>
<div class="flex items-center">
<flat-pickr :config="flatpickrConfig" :value="item.properties.created"
@ -72,9 +85,11 @@
</ul>
</div>
<button class="m-2 bg-green-500 hover:bg-green-600 text-white font-bold py-2 px-4 rounded"
@click="saveChanges">Save
</button>
<div v-if="itemsForUser.length > 0">
<button class="m-2 bg-green-500 hover:bg-green-600 text-white font-bold py-2 px-4 rounded"
@click="saveChanges">Save
</button>
</div>
<div class="hidden">
@ -107,6 +122,7 @@ export default {
return {
msg: "",
currentId: null,
originalFilename: null,
itemsForUser: [],
originalItems: [],
workerLog: [],
@ -114,7 +130,7 @@ export default {
enableTime: true,
time_24hr: true,
dateFormat: 'Y-m-d H:i',
timezone: 'UTC',
// timezone: 'UTC',
},
}
},
@ -200,6 +216,7 @@ export default {
} else {
vm.currentId = vm.id
if (Object.keys(response.data).length > 0) {
vm.originalFilename = response.data.original_filename
response.data.geofeatures.forEach((item) => {
vm.itemsForUser.push(vm.parseGeoJson(item))
})

View File

@ -1,20 +1,27 @@
<template>
<div class="mb-10">
<p>import data</p>
<p>Only KML/KMZ files supported.</p>
<p>Be careful not to upload duplicate files of the opposite type. For example, do not upload both
<kbd>example.kml</kbd>
and <kbd>example.kmz</kbd>. Currently, the system can't detect duplicate cross-file types.</p>
<p class="text-lg font-semibold mb-2">Import Data</p>
<p class="text-gray-600 mb-2">Only KML/KMZ files supported.</p>
<p class="text-gray-600">
Be careful not to upload duplicate files of the opposite type. For example, do not upload both
<kbd class="bg-gray-200 text-gray-800 px-2 py-1 rounded">example.kml</kbd>
and <kbd class="bg-gray-200 text-gray-800 px-2 py-1 rounded">example.kmz</kbd>. Currently, the system can't detect
duplicate cross-file types.
</p>
</div>
<div class="relative w-[90%] m-auto">
<div>
<input id="uploadInput" :disabled="disableUpload" type="file" @change="onFileChange">
<button :disabled="disableUpload" @click="upload">Upload</button>
<div class="relative w-[90%] mx-auto">
<div class="flex items-center">
<input id="uploadInput" :disabled="disableUpload" class="mr-4 px-4 py-2 border border-gray-300 rounded" type="file"
@change="onFileChange">
<button :disabled="disableUpload" class="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600 disabled:bg-gray-400 disabled:cursor-not-allowed"
@click="upload">
Upload
</button>
</div>
</div>
<div v-if="uploadMsg !== ''" class="w-[90%] m-auto mt-10" v-html="uploadMsg"></div>
<div v-if="uploadMsg !== ''" class="w-[90%] mx-auto mt-10" v-html="uploadMsg"></div>
<Importqueue/>
</template>
@ -66,7 +73,7 @@ export default {
formData.append('file', this.file)
try {
this.disableUpload = true
const response = await axios.post('/api/data/item/import/upload/', formData, {
const response = await axios.post('/api/data/item/import/upload', formData, {
headers: {
'Content-Type': 'multipart/form-data',
'X-CSRFToken': this.userInfo.csrftoken

View File

@ -1,30 +1,30 @@
<template>
<div>
<button @click="fetchQueueList">Refresh</button>
<div class="mt-4">
<button class="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600" @click="fetchQueueList">Refresh</button>
</div>
<table>
<table class="mt-6 w-full border-collapse">
<thead>
<tr>
<th>ID</th>
<th>File Name</th>
<th>Features</th>
<th></th>
<tr class="bg-gray-100">
<th class="px-4 py-2 text-left">File Name</th>
<th class="px-4 py-2 text-left">Features</th>
<th class="px-4 py-2"></th>
</tr>
</thead>
<tbody>
<tr v-for="(item, index) in importQueue" :key="`item-${index}`">
<td>
<a :href="`/#/import/process/${item.id}`">{{ item.id }}</a>
<tr v-for="(item, index) in importQueue" :key="`item-${index}`" class="border-t">
<td class="px-4 py-2">
<a :href="`/#/import/process/${item.id}`" class="text-blue-500 hover:text-blue-700">{{
item.original_filename
}}</a>
</td>
<td>
<a :href="`/#/import/process/${item.id}`">{{ item.original_filename }}</a>
</td>
<td>
<td class="px-4 py-2">
{{ item.processing === true ? "processing" : item.feature_count }}
</td>
<td>
<button @click="deleteItem(item, index)">Delete</button>
<td class="px-4 py-2">
<button class="px-4 py-2 bg-red-500 text-white rounded hover:bg-red-600" @click="deleteItem(item, index)">
Delete
</button>
</td>
</tr>
</tbody>

View File

@ -4,6 +4,8 @@ export default {
theme: {
extend: {},
},
plugins: [],
plugins: [
require('@tailwindcss/typography'),
],
}