export.py

@@ -2,26 +2,38 @@
 import json
 import os
 import string
-import traceback
 
 # external
 from canvasapi import Canvas
+from canvasapi.exceptions import ResourceDoesNotExist
 import dateutil.parser
 import jsonpickle
 import requests
+import yaml
+
+try:
+    with open("credentials.yaml", 'r') as f:
+        credentials = yaml.load(f)
+except OSError:
+    # Canvas API URL
+    API_URL = ""
+    # Canvas API key
+    API_KEY = ""
+    # My Canvas User ID
+    USER_ID = 0000000
+else:
+    API_URL = credentials["API_URL"]
+    API_KEY = credentials["API_KEY"]
+    USER_ID = credentials["USER_ID"]
 
-# Canvas API URL
-API_URL = ""
-# Canvas API key
-API_KEY = ""
-# My Canvas User ID
-USER_ID = 0000000
 # Directory in which to download course information to (will be created if not
 # present)
 DL_LOCATION = "./output"
 # List of Course IDs that should be skipped (need to be integers)
 COURSES_TO_SKIP = [288290, 512033]
 
+DATE_TEMPLATE = "%B %d, %Y %I:%M %p"
+
 
 class moduleItemView():
     title = ""
@@ -72,29 +84,42 @@ class discussionView():
 
 
 class submissionView():
+    attachments = []
     grade = ""
     raw_score = ""
-    total_possible_points = ""
     submission_comments = ""
-    user_id = None  # integer
-    attachments = []
+    total_possible_points = ""
+    user_id = "no-id"
+
+    def __init__(self):
+        self.attachments = []
+        self.grade = ""
+        self.raw_score = ""
+        self.submission_comments = ""
+        self.total_possible_points = ""
+        self.user_id = None  # integer
 
 
 class attachmentView():
     filename = ""
     url = ""
 
+    def __init__(self):
+        self.filename = ""
+        self.url = ""
+
 
 class assignmentView():
     title = ""
     description = ""
     assigned_date = ""
     due_date = ""
-    submissions = {}
     submission = None
+    submissions = []
 
     def __init__(self):
         self.submission = submissionView()
+        self.submissions = []
 
 
 class courseView():
@@ -123,7 +148,8 @@ def makeValidFilename(input_str):
 
 
 def findCourseModules(course, course_view):
-    modules_dir = DL_LOCATION + "/" + course_view.term + "/" + course_view.course_code + "/modules"
+    modules_dir = os.path.join(DL_LOCATION, course_view.term,
+                               course_view.course_code, "modules")
 
     # Create modules directory if not present
     if not os.path.exists(modules_dir):
@@ -190,8 +216,10 @@ def findCourseModules(course, course_view):
 
     return module_views
 
+
 def downloadCourseFiles(course, course_view):
-    dl_dir = DL_LOCATION + "/" + course_view.term + "/" + course_view.course_code + "/files"
+    dl_dir = os.path.join(DL_LOCATION, course_view.term,
+                          course_view.course_code, "files")
 
     # Create directory if not present
     if not os.path.exists(dl_dir):
@@ -201,7 +229,8 @@ def downloadCourseFiles(course, course_view):
         files = course.get_files()
 
         for file in files:
-            dl_path = dl_dir + "/" + makeValidFilename(str(file.display_name))
+            dl_path = os.path.join(dl_dir,
+                                   makeValidFilename(str(file.display_name)))
 
             # Download file if it doesn't already exist
             if not os.path.exists(dl_path):
@@ -211,6 +240,30 @@ def downloadCourseFiles(course, course_view):
         print("Skipping file download that gave the following error:")
         print(e)
 
+
+def download_submission_attachments(course, course_view):
+    course_dir = os.path.join(DL_LOCATION, course_view.term,
+                              course_view.course_code)
+
+    # Create directory if not present
+    if not os.path.exists(course_dir):
+        os.makedirs(course_dir)
+
+    for assignment in course_view.assignments:
+        for submission in assignment.submissions:
+            attachment_dir = os.path.join(course_dir, assignment.title,
+                                          str(submission.user_id))
+            if not os.path.exists(attachment_dir):
+                os.makedirs(attachment_dir)
+            for attachment in submission.attachments:
+                filepath = os.path.join(attachment_dir, attachment.filename)
+                if not os.path.exists(filepath):
+                    print('Downloading attachment: {}'.format(attachment.filename))
+                    r = requests.get(attachment.url, allow_redirects=True)
+                    with open(filepath, 'wb') as f:
+                        f.write(r.content)
+
+
 def getCoursePageUrls(course):
     page_urls = []
 
@@ -228,6 +281,7 @@ def getCoursePageUrls(course):
 
     return page_urls
 
+
 def findCoursePages(course):
     page_views = []
 
@@ -245,9 +299,17 @@ def findCoursePages(course):
             # Body
             page_view.body = str(page.body) if hasattr(page, "body") else ""
             # Date created
-            page_view.created_date = dateutil.parser.parse(page.created_at).strftime("%B %d, %Y %I:%M %p") if hasattr(page, "created_at") else ""
+            if hasattr(page, "created_at"):
+                page_view.created_date = dateutil.parser.parse(
+                    page.created_at).strftime(DATE_TEMPLATE)
+            else:
+                page_view.created_date = ""
             # Date last updated
-            page_view.last_updated_date = dateutil.parser.parse(page.updated_at).strftime("%B %d, %Y %I:%M %p") if hasattr(page, "updated_at") else ""
+            if hasattr(page, "updated_at"):
+                page_view.last_updated_date = dateutil.parser.parse(
+                    page.updated_at).strftime(DATE_TEMPLATE)
+            else:
+                page_view.last_updated_date = ""
 
             page_views.append(page_view)
     except Exception as e:
@@ -256,52 +318,94 @@ def findCoursePages(course):
 
     return page_views
 
+
 def findCourseAssignments(course):
     assignment_views = []
 
-    try:
-        # Get all assignments
-        assignments = course.get_assignments()
+    # Get all assignments
+    assignments = course.get_assignments()
 
-        for assignment in assignments:
-            print(assignment)
-            # Create a new assignment view
-            assignment_view = assignmentView()
+    for assignment in assignments:
+        # Create a new assignment view
+        assignment_view = assignmentView()
 
-            # Title
-            assignment_view.title = str(assignment.name) if hasattr(assignment, "name") else ""
-            # Description
-            assignment_view.description = str(assignment.description) if hasattr(assignment, "description") else ""
-            # Assigned date
-            assignment_view.assigned_date = assignment.created_at_date.strftime("%B %d, %Y %I:%M %p") if hasattr(assignment, "created_at_date") else ""
-            # Due date
-            assignment_view.due_date = assignment.due_at_date.strftime("%B %d, %Y %I:%M %p") if hasattr(assignment, "due_at_date") else ""
+        # Title
+        if hasattr(assignment, "name"):
+            assignment_view.title = str(assignment.name)
+        else:
+            assignment_view.title = ""
+        # Description
+        if hasattr(assignment, "description"):
+            assignment_view.description = str(assignment.description)
+        else:
+            assignment_view.description = ""
+        # Assigned date
+        if hasattr(assignment, "created_at_date"):
+            assignment_view.assigned_date = assignment.created_at_date.strftime(DATE_TEMPLATE)
+        else:
+            assignment_view.assigned_date = ""
+        # Due date
+        if hasattr(assignment, "due_at_date"):
+            assignment_view.due_date = assignment.due_at_date.strftime(DATE_TEMPLATE)
+        else:
+            assignment_view.due_date = ""
 
-            # Download all submissions
-            try:
-                submissions = assignment.get_submissions()
-            except:
-                print("Got no submissions for this assignment")
-            else:
-                print(submissions)
-                for submission in submissions:
-                    print(submission)
-                    sub_view = submissionView()
-                    try:
-                        submission.attachments
-                    except AttributeError:
-                        print('No attachments')
-                    else:
-                        for attachment in submission.attachments:
-                            attach_view = attachmentView()
-                            attach_view.url = attachment.url
-                            attach_view.filename = attachment.filename
-                            sub_view.attachments.append
-                            print(attachment["url"])
+        # Download all submissions
+        try:
+            submissions = assignment.get_submissions()
+        # TODO : Figure out the exact error raised
+        except:
+            print("Got no submissions for this assignment")
+        else:
+            for submission in submissions:
 
-            # Get my user"s submission object
+                sub_view = submissionView()
+
+                # My grade
+                if hasattr(submission, "grade"):
+                    sub_view.grade = str(submission.grade)
+                else:
+                    sub_view.grade = ""
+                # My raw score
+                if hasattr(submission, "score"):
+                    sub_view.raw_score = str(submission.score)
+                else:
+                    sub_view.raw_score = ""
+                # Total possible score
+                if hasattr(assignment, "points_possible"):
+                    sub_view.total_possible_points = str(assignment.points_possible)
+                else:
+                    sub_view.total_possible_points = ""
+                # Submission comments
+                if hasattr(submission, "submission_comments"):
+                    sub_view.submission_comments = str(submission.submission_comments)
+                else:
+                    sub_view.submission_comments = ""
+
+                if hasattr(submission, "user_id"):
+                    sub_view.user_id = str(submission.user_id)
+                else:
+                    sub_view.user_id = "no-id"
+
+                try:
+                    submission.attachments
+                except AttributeError:
+                    print('No attachments')
+                else:
+                    for attachment in submission.attachments:
+                        attach_view = attachmentView()
+                        attach_view.url = attachment["url"]
+                        attach_view.filename = attachment["filename"]
+                        sub_view.attachments.append(attach_view)
+                assignment_view.submissions.append(sub_view)
+
+        # The following is only useful if you are a student in the class.
+        # Get my user"s submission object
+        try:
             submission = assignment.get_submission(USER_ID)
-
+        except ResourceDoesNotExist:
+            print('No submission for user: {}'.format(USER_ID))
+        else:
             # Create a new submission view
             assignment_view.submission = submissionView()
 
@@ -314,16 +418,12 @@ def findCourseAssignments(course):
             # Submission comments
             assignment_view.submission.submission_comments = str(submission.submission_comments) if hasattr(submission, "submission_comments") else ""
 
-            assignment_views.append(assignment_view)
+        assignment_views.append(assignment_view)
 
 
-
-    except Exception as e:
-        print("Skipping assignment that gave the following error:")
-        print(e)
-
     return assignment_views
 
+
 def findCourseAnnouncements(course):
     announcement_views = []
 
@@ -340,6 +440,7 @@ def findCourseAnnouncements(course):
 
     return announcement_views
 
+
 def getDiscussionView(discussion_topic):
     # Create discussion view
     discussion_view = discussionView()
@@ -397,6 +498,7 @@ def getDiscussionView(discussion_topic):
 
     return discussion_view
 
+
 def findCourseDiscussions(course):
     discussion_views = []
 
@@ -414,6 +516,7 @@ def findCourseDiscussions(course):
 
     return discussion_views
 
+
 def getCourseView(course):
     course_view = courseView()
 
@@ -446,37 +549,46 @@ def getCourseView(course):
 
     return course_view
 
+
 def exportAllCourseData(course_view):
     json_str = json.dumps(json.loads(jsonpickle.encode(course_view, unpicklable = False)), indent = 4)
 
-    course_output_dir = DL_LOCATION + "/" + course_view.term + "/" + course_view.course_code
+    course_output_dir = os.path.join(DL_LOCATION, course_view.term,
+                                     course_view.course_code)
 
     # Create directory if not present
     if not os.path.exists(course_output_dir):
         os.makedirs(course_output_dir)
 
-    course_output_path = course_output_dir + "/" + course_view.course_code + ".json"
+    course_output_path = os.path.join(course_output_dir,
+                                      course_view.course_code + ".json")
 
     with open(course_output_path, "w") as out_file:
         out_file.write(json_str)
 
-def main():
+
+if __name__ == "__main__":
+
     print("Welcome to the Canvas Student Data Export Tool\n")
 
-    # Canvas API URL
-    print("We will need your organization's Canvas Base URL. This is probably something like https://{schoolName}.instructure.com)")
-    global API_URL
-    #API_URL = input("Enter your organization's Canvas Base URL: ")
+    if API_URL == "":
+        # Canvas API URL
+        print("We will need your organization's Canvas Base URL. This is "
+              "probably something like https://{schoolName}.instructure.com)")
+        API_URL = input("Enter your organization's Canvas Base URL: ")
 
-    # Canvas API key
-    print("\nWe will need a valid API key for your user. You can generate one in Canvas once you are logged in.")
-    global API_KEY
-    #API_KEY = input("Enter a valid API key for your user: ")
+    if API_KEY == "":
+        # Canvas API key
+        print("\nWe will need a valid API key for your user. You can generate "
+              "one in Canvas once you are logged in.")
+        API_KEY = input("Enter a valid API key for your user: ")
 
-    # My Canvas User ID
-    print("\nWe will need your Canvas User ID. You can find this by logging in to canvas and then going to this URL in the same browser {yourCanvasBaseUrl}/api/v1/users/self")
-    global USER_ID
-    #USER_ID = input("Enter your Canvas User ID: ")
+    if USER_ID == 0000000:
+        # My Canvas User ID
+        print("\nWe will need your Canvas User ID. You can find this by "
+              "logging in to canvas and then going to this URL in the same "
+              "browser {yourCanvasBaseUrl}/api/v1/users/self")
+        USER_ID = input("Enter your Canvas User ID: ")
 
     print("\nConnecting to canvas\n")
 
@@ -490,48 +602,42 @@ def main():
 
     all_courses_views = []
 
-    try:
-        print("Getting list of all courses\n")
-        courses = canvas.get_courses(include="term")
+    print("Getting list of all courses\n")
+    courses = canvas.get_courses(include="term")
 
-        # I am not authorized to access course 1083083
-        skip = set(COURSES_TO_SKIP)
+    skip = set(COURSES_TO_SKIP)
 
-        for course in courses:
-            if course.id in skip:
-                continue
+    for course in courses:
+        if course.id in skip:
+            continue
 
-            course_view = getCourseView(course)
+        course_view = getCourseView(course)
 
-            all_courses_views.append(course_view)
+        all_courses_views.append(course_view)
 
-            print("  Downloading all files")
-            downloadCourseFiles(course, course_view)
+        print("  Downloading all files")
+        downloadCourseFiles(course, course_view)
 
-            print("  Getting modules and downloading module files")
-            course_view.modules = findCourseModules(course, course_view)
+        print("  Downloading submission attachments")
+        download_submission_attachments(course, course_view)
 
-            print("  Exporting all course data")
-            exportAllCourseData(course_view)
-    except Exception as e:
-        print("Skipping entire course that gave the following error:")
-        print(e)
+        print("  Getting modules and downloading module files")
+        course_view.modules = findCourseModules(course, course_view)
 
-    print("Exporting data from all courses combined as one file: all_output.json")
-    # Awful hack to make the JSON pretty. Decode it with Python stdlib json module then re-encode with indentation
-    json_str = json.dumps(json.loads(jsonpickle.encode(all_courses_views, unpicklable = False)), indent = 4)
+        print("  Exporting all course data")
+        exportAllCourseData(course_view)
 
-    all_output_path = DL_LOCATION + "/all_output.json"
+    print("Exporting data from all courses combined as one file: "
+          "all_output.json")
+    # Awful hack to make the JSON pretty. Decode it with Python stdlib json
+    # module then re-encode with indentation
+    json_str = json.dumps(json.loads(jsonpickle.encode(all_courses_views,
+                                                       unpicklable=False)),
+                          indent=4)
+
+    all_output_path = os.path.join(DL_LOCATION, "all_output.json")
 
     with open(all_output_path, "w") as out_file:
         out_file.write(json_str)
 
     print("\nProcess complete. All canvas data exported!")
-
-if __name__ == "__main__":
-    try:
-        main()
-    except Exception as e:
-        print("Exiting due to uncaught exception:")
-        print(e)
-        print(traceback.format_exc())