cyberes
/
canvas-student-data-export
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added support to download canvas webpages #7

Merged
dj346 merged 16 commits from master into master 2022-01-22 10:21:05 -07:00
Conversation 0 Commits 16 Files Changed 6 +153 -71
2 changed files with 153 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit c9a3424330 - Show all commits

2
.gitignore vendored
View File

@ -1,4 +1,6 @@
.vscode .vscode
output/ output/
node_modules/ node_modules/
credentials.yaml credentials.yaml
cookies.txt

222
export.py
View File

@ -5,8 +5,8 @@ import string
# external # external
from canvasapi import Canvas from canvasapi import Canvas
from canvasapi.exceptions import ResourceDoesNotExist from canvasapi.exceptions import ResourceDoesNotExist, Unauthorized
from canvasapi.exceptions import Unauthorized
import dateutil.parser import dateutil.parser
import jsonpickle import jsonpickle
import requests import requests
@ -22,10 +22,13 @@ except OSError:
API_KEY = "" API_KEY = ""
# My Canvas User ID # My Canvas User ID
USER_ID = 0000000 USER_ID = 0000000
# Browser Cookies File
COOKIES_PATH = ""
else: else:
API_URL = credentials["API_URL"] API_URL = credentials["API_URL"]
API_KEY = credentials["API_KEY"] API_KEY = credentials["API_KEY"]
USER_ID = credentials["USER_ID"] USER_ID = credentials["USER_ID"]
COOKIES_PATH = credentials["COOKIES_PATH"]
# Directory in which to download course information to (will be created if not # Directory in which to download course information to (will be created if not
# present) # present)
@ -37,12 +40,18 @@ DATE_TEMPLATE = "%B %d, %Y %I:%M %p"
class moduleItemView(): class moduleItemView():
id = 0
title = "" title = ""
content_type = "" content_type = ""
url = ""
external_url = "" external_url = ""
class moduleView(): class moduleView():
id = 0
name = "" name = ""
items = [] items = []
@ -51,6 +60,8 @@ class moduleView():
class pageView(): class pageView():
id = 0
title = "" title = ""
body = "" body = ""
created_date = "" created_date = ""
@ -58,12 +69,16 @@ class pageView():
class topicReplyView(): class topicReplyView():
id = 0
author = "" author = ""
posted_date = "" posted_date = ""
body = "" body = ""
class topicEntryView(): class topicEntryView():
id = 0
author = "" author = ""
posted_date = "" posted_date = ""
body = "" body = ""
@ -74,70 +89,82 @@ class topicEntryView():
class discussionView(): class discussionView():
id = 0
title = "" title = ""
author = "" author = ""
posted_date = "" posted_date = ""
body = "" body = ""
topic_entries = [] topic_entries = []
url = ""
amount_pages = 0
def __init__(self): def __init__(self):
self.topic_entries = [] self.topic_entries = []
class submissionView(): class submissionView():
id = 0
attachments = [] attachments = []
grade = "" grade = ""
raw_score = "" raw_score = ""
submission_comments = "" submission_comments = ""
total_possible_points = "" total_possible_points = ""
attempt = 0
user_id = "no-id" user_id = "no-id"
preview_url = ""
ext_url = ""
def __init__(self): def __init__(self):
self.attachments = [] self.attachments = []
self.grade = ""
self.raw_score = ""
self.submission_comments = ""
self.total_possible_points = ""
self.user_id = None # integer
class attachmentView(): class attachmentView():
filename = ""
id = 0 id = 0
filename = ""
url = "" url = ""
def __init__(self):
self.filename = ""
self.id = 0
self.url = ""
class assignmentView(): class assignmentView():
id = 0
title = "" title = ""
description = "" description = ""
assigned_date = "" assigned_date = ""
due_date = "" due_date = ""
submissions = [] submissions = []
html_url = ""
ext_url = ""
updated_url = ""
def __init__(self): def __init__(self):
self.submissions = [] self.submissions = []
class courseView(): class courseView():
course_id = 0
term = "" term = ""
course_code = "" course_code = ""
name = "" name = ""
assignments = [] assignments = []
announcements = [] announcements = []
discussions = [] discussions = []
modules = []
def __init__(self): def __init__(self):
self.assignments = [] self.assignments = []
self.announcements = [] self.announcements = []
self.discussions = [] self.discussions = []
self.modules = []
def makeValidFilename(input_str): def makeValidFilename(input_str):
if(not input_str):
return input_str
# Remove invalid characters # Remove invalid characters
valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
input_str = input_str.replace("+"," ") # Canvas default for spaces input_str = input_str.replace("+"," ") # Canvas default for spaces
@ -148,6 +175,9 @@ def makeValidFilename(input_str):
# Remove leading and trailing whitespace # Remove leading and trailing whitespace
input_str = input_str.lstrip().rstrip() input_str = input_str.lstrip().rstrip()
##Splits strings to prevent extremely long names
#input_str=input_str[:40]
return input_str return input_str
def makeValidFolderPath(input_str): def makeValidFolderPath(input_str):
@ -162,7 +192,10 @@ def makeValidFolderPath(input_str):
# Replace path separators with OS default # Replace path separators with OS default
input_str=input_str.replace("/",os.sep) input_str=input_str.replace("/",os.sep)
##Splits strings to prevent extremely long names
#input_str=input_str[:40]
return input_str return input_str
def findCourseModules(course, course_view): def findCourseModules(course, course_view):
@ -181,6 +214,9 @@ def findCourseModules(course, course_view):
for module in modules: for module in modules:
module_view = moduleView() module_view = moduleView()
# ID
module_view.id = module.id if hasattr(module, "id") else ""
# Name # Name
module_view.name = str(module.name) if hasattr(module, "name") else "" module_view.name = str(module.name) if hasattr(module, "name") else ""
@ -191,17 +227,23 @@ def findCourseModules(course, course_view):
for module_item in module_items: for module_item in module_items:
module_item_view = moduleItemView() module_item_view = moduleItemView()
# ID
module_item_view.id = module_item.id if hasattr(module_item, "id") else 0
# Title # Title
module_item_view.title = str(module_item.title) if hasattr(module_item, "title") else "" module_item_view.title = str(module_item.title) if hasattr(module_item, "title") else ""
# Type # Type
module_item_view.content_type = str(module_item.type) if hasattr(module_item, "type") else "" module_item_view.content_type = str(module_item.type) if hasattr(module_item, "type") else ""
# URL
module_item_view.url = str(module_item.html_url) if hasattr(module_item, "html_url") else ""
# External URL # External URL
module_item_view.external_url = str(module_item.external_url) if hasattr(module_item, "external_url") else "" module_item_view.external_url = str(module_item.external_url) if hasattr(module_item, "external_url") else ""
if module_item_view.content_type == "File": if module_item_view.content_type == "File":
module_dir = modules_dir + "/" + makeValidFilename(str(module.name)) # If problems arise due to long pathnames, changing module.name to module.id might help
# A change would also have to be made in downloadCourseModulePages(api_url, course_view, cookies_path)
module_dir = os.path.join(modules_dir, makeValidFolderPath(str(module.id)), "files")
try: try:
# Create directory for current module if not present # Create directory for current module if not present
@ -212,7 +254,7 @@ def findCourseModules(course, course_view):
module_file = course.get_file(str(module_item.content_id)) module_file = course.get_file(str(module_item.content_id))
# Create path for module file download # Create path for module file download
module_file_path = module_dir + "/" + makeValidFilename(str(module_file.display_name)) module_file_path = os.path.join(module_dir, makeValidFilename(str(module_file.display_name)))
# Download file if it doesn't already exist # Download file if it doesn't already exist
if not os.path.exists(module_file_path): if not os.path.exists(module_file_path):
@ -250,13 +292,12 @@ def downloadCourseFiles(course, course_view):
for file in files: for file in files:
file_folder=course.get_folder(file.folder_id) file_folder=course.get_folder(file.folder_id)
folder_dl_dir=os.path.join(dl_dir,makeValidFolderPath(file_folder.full_name)) folder_dl_dir=os.path.join(dl_dir, makeValidFolderPath(file_folder.full_name))
if not os.path.exists(folder_dl_dir): if not os.path.exists(folder_dl_dir):
os.makedirs(folder_dl_dir) os.makedirs(folder_dl_dir)
dl_path = os.path.join(folder_dl_dir, dl_path = os.path.join(folder_dl_dir, makeValidFilename(str(file.display_name)))
davekats commented 2022-01-13 22:08:05 -07:00 (Migrated from github.com)
Review
Copy Link

Could we remove or merge the two validation functions?

Could we remove or merge the two validation functions?
dj346 commented 2022-01-13 23:05:36 -07:00 (Migrated from github.com)
Review
Copy Link

Oops I was testing out whether makeValidFolderPath was needed and I must've left that changed. I think in that case it might make sense to keep it makeValidFolderPath.

Oops I was testing out whether makeValidFolderPath was needed and I must've left that changed. I think in that case it might make sense to keep it makeValidFolderPath.
davekats commented 2022-01-14 00:00:18 -07:00 (Migrated from github.com)
Review
Copy Link

Sounds good.

Sounds good.
makeValidFilename(str(file.display_name)))
# Download file if it doesn't already exist # Download file if it doesn't already exist
if not os.path.exists(dl_path): if not os.path.exists(dl_path):
@ -324,22 +365,19 @@ def findCoursePages(course):
page_view = pageView() page_view = pageView()
# ID
page_view.id = page.id if hasattr(page, "id") else 0
# Title # Title
page_view.title = str(page.title) if hasattr(page, "title") else "" page_view.title = str(page.title) if hasattr(page, "title") else ""
# Body # Body
page_view.body = str(page.body) if hasattr(page, "body") else "" page_view.body = str(page.body) if hasattr(page, "body") else ""
# Date created # Date created
if hasattr(page, "created_at"): page_view.created_date = dateutil.parser.parse(page.created_at).strftime(DATE_TEMPLATE) if \
page_view.created_date = dateutil.parser.parse( hasattr(page, "created_at") else ""
page.created_at).strftime(DATE_TEMPLATE)
else:
page_view.created_date = ""
# Date last updated # Date last updated
if hasattr(page, "updated_at"): page_view.last_updated_date = dateutil.parser.parse(page.updated_at).strftime(DATE_TEMPLATE) if \
page_view.last_updated_date = dateutil.parser.parse( hasattr(page, "updated_at") else ""
page.updated_at).strftime(DATE_TEMPLATE)
else:
page_view.last_updated_date = ""
page_views.append(page_view) page_views.append(page_view)
except Exception as e: except Exception as e:
@ -360,26 +398,33 @@ def findCourseAssignments(course):
# Create a new assignment view # Create a new assignment view
assignment_view = assignmentView() assignment_view = assignmentView()
#ID
assignment_view.id = assignment.id if \
hasattr(assignment, "id") else ""
# Title # Title
if hasattr(assignment, "name"): assignment_view.title = makeValidFilename(str(assignment.name)) if \
assignment_view.title = makeValidFilename(str(assignment.name)) hasattr(assignment, "name") else ""
else:
assignment_view.title = ""
# Description # Description
if hasattr(assignment, "description"): assignment_view.description = str(assignment.description) if \
assignment_view.description = str(assignment.description) hasattr(assignment, "description") else ""
else:
assignment_view.description = ""
# Assigned date # Assigned date
if hasattr(assignment, "created_at_date"): assignment_view.assigned_date = assignment.created_at_date.strftime(DATE_TEMPLATE) if \
assignment_view.assigned_date = assignment.created_at_date.strftime(DATE_TEMPLATE) hasattr(assignment, "created_at_date") else ""
else:
assignment_view.assigned_date = ""
# Due date # Due date
if hasattr(assignment, "due_at_date"): assignment_view.due_date = assignment.due_at_date.strftime(DATE_TEMPLATE) if \
assignment_view.due_date = assignment.due_at_date.strftime(DATE_TEMPLATE) hasattr(assignment, "due_at_date") else ""
else:
assignment_view.due_date = "" # HTML Url
assignment_view.html_url = assignment.html_url if \
hasattr(assignment, "html_url") else ""
# External URL
assignment_view.ext_url = str(assignment.url) if \
hasattr(assignment, "url") else ""
# Other URL (more up-to-date)
assignment_view.updated_url = str(assignment.submissions_download_url).split("submissions?")[0] if \
hasattr(assignment, "submissions_download_url") else ""
try: try:
try: # Download all submissions for entire class try: # Download all submissions for entire class
@ -401,31 +446,35 @@ def findCourseAssignments(course):
sub_view = submissionView() sub_view = submissionView()
# Submission ID
sub_view.id = submission.id if \
hasattr(submission, "id") else 0
# My grade # My grade
if hasattr(submission, "grade"): sub_view.grade = str(submission.grade) if \
sub_view.grade = str(submission.grade) hasattr(submission, "grade") else ""
else:
sub_view.grade = ""
# My raw score # My raw score
if hasattr(submission, "score"): sub_view.raw_score = str(submission.score) if \
sub_view.raw_score = str(submission.score) hasattr(submission, "score") else ""
else:
sub_view.raw_score = ""
# Total possible score # Total possible score
if hasattr(assignment, "points_possible"): sub_view.total_possible_points = str(assignment.points_possible) if \
sub_view.total_possible_points = str(assignment.points_possible) hasattr(assignment, "points_possible") else ""
else:
sub_view.total_possible_points = ""
# Submission comments # Submission comments
if hasattr(submission, "submission_comments"): sub_view.submission_comments = str(submission.submission_comments) if \
sub_view.submission_comments = str(submission.submission_comments) hasattr(submission, "submission_comments") else ""
else: # Attempt
sub_view.submission_comments = "" sub_view.attempt = submission.attempt if \
hasattr(submission, "attempt") else 0
if hasattr(submission, "user_id"): # User ID
sub_view.user_id = str(submission.user_id) sub_view.user_id = str(submission.user_id) if \
else: hasattr(submission, "user_id") else ""
sub_view.user_id = "no-id"
# Submission URL
sub_view.preview_url = str(submission.preview_url) if \
hasattr(submission, "preview_url") else ""
# External URL
sub_view.ext_url = str(submission.url) if \
hasattr(submission, "url") else ""
try: try:
submission.attachments submission.attachments
@ -472,6 +521,9 @@ def getDiscussionView(discussion_topic):
# Create discussion view # Create discussion view
discussion_view = discussionView() discussion_view = discussionView()
#ID
discussion_view.id = discussion_topic.id if hasattr(discussion_topic, "id") else 0
# Title # Title
discussion_view.title = str(discussion_topic.title) if hasattr(discussion_topic, "title") else "" discussion_view.title = str(discussion_topic.title) if hasattr(discussion_topic, "title") else ""
# Author # Author
@ -480,6 +532,13 @@ def getDiscussionView(discussion_topic):
discussion_view.posted_date = discussion_topic.created_at_date.strftime("%B %d, %Y %I:%M %p") if hasattr(discussion_topic, "created_at_date") else "" discussion_view.posted_date = discussion_topic.created_at_date.strftime("%B %d, %Y %I:%M %p") if hasattr(discussion_topic, "created_at_date") else ""
# Body # Body
discussion_view.body = str(discussion_topic.message) if hasattr(discussion_topic, "message") else "" discussion_view.body = str(discussion_topic.message) if hasattr(discussion_topic, "message") else ""
# URL
discussion_view.url = str(discussion_topic.html_url) if hasattr(discussion_topic, "html_url") else ""
# Keeps track of how many topic_entries there are.
topic_entries_counter = 0
# Topic entries # Topic entries
if hasattr(discussion_topic, "discussion_subentry_count") and discussion_topic.discussion_subentry_count > 0: if hasattr(discussion_topic, "discussion_subentry_count") and discussion_topic.discussion_subentry_count > 0:
# Need to get replies to entries recursively? # Need to get replies to entries recursively?
@ -488,9 +547,13 @@ def getDiscussionView(discussion_topic):
try: try:
for topic_entry in discussion_topic_entries: for topic_entry in discussion_topic_entries:
topic_entries_counter += 1
# Create new discussion view for the topic_entry # Create new discussion view for the topic_entry
topic_entry_view = topicEntryView() topic_entry_view = topicEntryView()
# ID
topic_entry_view.id = topic_entry.id if hasattr(topic_entry, "id") else 0
# Author # Author
topic_entry_view.author = str(topic_entry.user_name) if hasattr(topic_entry, "user_name") else "" topic_entry_view.author = str(topic_entry.user_name) if hasattr(topic_entry, "user_name") else ""
# Posted date # Posted date
@ -505,6 +568,9 @@ def getDiscussionView(discussion_topic):
for topic_reply in topic_entry_replies: for topic_reply in topic_entry_replies:
# Create new topic reply view # Create new topic reply view
topic_reply_view = topicReplyView() topic_reply_view = topicReplyView()
# ID
topic_reply_view.id = topic_reply.id if hasattr(topic_reply, "id") else 0
# Author # Author
topic_reply_view.author = str(topic_reply.user_name) if hasattr(topic_reply, "user_name") else "" topic_reply_view.author = str(topic_reply.user_name) if hasattr(topic_reply, "user_name") else ""
@ -522,7 +588,10 @@ def getDiscussionView(discussion_topic):
except Exception as e: except Exception as e:
print("Tried to enumerate discussion topic entries but received the following error:") print("Tried to enumerate discussion topic entries but received the following error:")
print(e) print(e)
# Amount of pages
discussion_view.amount_pages = int(topic_entries_counter/50) + 1 # Typically 50 topic entries are stored on a page before it creates another page.
return discussion_view return discussion_view
@ -547,6 +616,9 @@ def findCourseDiscussions(course):
def getCourseView(course): def getCourseView(course):
course_view = courseView() course_view = courseView()
# Course ID
course_view.course_id = course.id if hasattr(course, "id") else 0
# Course term # Course term
course_view.term = makeValidFilename(course.term["name"] if hasattr(course, "term") and "name" in course.term.keys() else "") course_view.term = makeValidFilename(course.term["name"] if hasattr(course, "term") and "name" in course.term.keys() else "")
@ -616,6 +688,14 @@ if __name__ == "__main__":
"logging in to canvas and then going to this URL in the same " "logging in to canvas and then going to this URL in the same "
"browser {yourCanvasBaseUrl}/api/v1/users/self") "browser {yourCanvasBaseUrl}/api/v1/users/self")
USER_ID = input("Enter your Canvas User ID: ") USER_ID = input("Enter your Canvas User ID: ")
if COOKIES_PATH == "":
# Cookies path
print("\nWe will need your browsers cookies file. This needs to be "
"exported using another tool. This needs to be a path to a file "
"formatted in the NetScape format. This can be left blank if an html "
"images aren't wanted. ")
COOKIES_PATH = input("Enter your cookies path: ")
print("\nConnecting to canvas\n") print("\nConnecting to canvas\n")