#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Complete the 'extract_airports' function so that it returns a list of airport
codes, excluding any combinations like "All".
"""
from bs4 import BeautifulSoup
html_page = "./data/options.html"
def extract_airports(page):
data = []
with open(page, "r") as html:
soup = BeautifulSoup(html, "lxml")
AirportList = soup.find(id='AirportList')
for airport in AirportList.find_all('option'):
if airport['value'][0:3] != 'All':
data.append(airport['value'])
return data
def test():
data = extract_airports(html_page)
assert len(data) == 15
assert "ATL" in data
assert "ABR" in data
test()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Let's assume that you combined the code from the previous 2 exercises with code
from the lesson on how to build requests, and downloaded all the data locally.
The files are in a directory "data", named after the carrier and airport:
"{}-{}.html".format(carrier, airport), for example "FL-ATL.html".
The table with flight info has a table class="dataTDRight". Your task is to
extract the flight data from that table as a list of dictionaries, each
dictionary containing relevant data from the file and table row. This is an
example of the data structure you should return:
data = [{"courier": "FL",
"airport": "ATL",
"year": 2012,
"month": 12,
"flights": {"domestic": 100,
"international": 100}
},
{"courier": "..."}
]
Note - year, month, and the flight data should be integers.
You should skip the rows that contain the TOTAL data for a year.
There are couple of helper functions to deal with the data files.
Please do not change them for grading purposes.
All your changes should be in the 'process_file' function.
"""
from bs4 import BeautifulSoup
from zipfile import ZipFile
import os
datadir = "data"
def open_zip(datadir):
with ZipFile('{0}.zip'.format(datadir), 'r') as myzip:
myzip.extractall()
def process_all(datadir):
#files = os.listdir(datadir)
files = ['FL-ATL.html']
return files
def process_file(f):
"""
This function extracts data from the file given as the function argument in
a list of dictionaries. This is example of the data structure you should
return:
data = [{"courier": "FL",
"airport": "ATL",
"year": 2012,
"month": 12,
"flights": {"domestic": 100,
"international": 100}
},
{"courier": "..."}
]
Note - year, month, and the flight data should be integers.
You should skip the rows that contain the TOTAL data for a year.
"""
data = []
info = {}
info["courier"], info["airport"] = f[:6].split("-")
# Note: create a new dictionary for each entry in the output data list.
# If you use the info dictionary defined here each element in the list
# will be a reference to the same info dictionary.
with open("{}/{}".format(datadir, f), "r") as html:
soup = BeautifulSoup(html,'lxml')
#flight_table = soup.find('table', class='dataTDRight')
rows = soup.find_all('tr', 'dataTDRight')
for row in rows:
tds = row.find_all('td')
if tds[1].text != 'TOTAL':
info['year'] = int(tds[0].text)
info['month'] = int(tds[1].text)
info['flights'] = {'domestic': int(tds[2].text.replace(',','')),
'international': int(tds[3].text.replace(',',''))}
data.append(info.copy())
return data
def test():
print "Running a simple test..."
#open_zip(datadir)
files = process_all(datadir)
data = []
# Test will loop over three data files.
for f in files:
data += process_file(f)
#assert len(data) == 399 # Total number of rows
for entry in data[:3]:
assert type(entry["year"]) == int
assert type(entry["month"]) == int
assert type(entry["flights"]["domestic"]) == int
assert len(entry["airport"]) == 3
assert len(entry["courier"]) == 2
assert data[0]["courier"] == 'FL'
assert data[0]["month"] == 10
#assert data[-1]["airport"] == "ATL"
#assert data[-1]["flights"] == {'international': 108289, 'domestic': 701425}
print "... success!"
if __name__ == "__main__":
test()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This and the following exercise are using US Patent database. The patent.data
file is a small excerpt of much larger datafiles that are available for
download from US Patent website. These files are pretty large ( >100 MB each).
The original file is ~600MB large, you might not be able to open it in a text
editor.
The data itself is in XML, however there is a problem with how it's formatted.
Please run this script and observe the error. Then find the line that is
causing the error. You can do that by just looking at the datafile in the web
UI, or programmatically. For quiz purposes it does not matter, but as an
exercise we suggest that you try to do it programmatically.
NOTE: You do not need to correct the error - for now, just find where the error
is occurring.
"""
import xml.etree.ElementTree as ET
PATENTS = './data/patent.data'
def get_root(fname):
tree = ET.parse(fname)
return tree.getroot()
get_root(PATENTS)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# So, the problem is that the gigantic file is actually not a valid XML, because
# it has several root elements, and XML declarations.
# It is, a matter of fact, a collection of a lot of concatenated XML documents.
# So, one solution would be to split the file into separate documents,
# so that you can process the resulting files as valid XML documents.
import xml.etree.ElementTree as ET
PATENTS = './data/patent.data'
def get_root(fname):
tree = ET.parse(fname)
return tree.getroot()
### from http://stackoverflow.com/questions/12717294/split-one-file-into-multiple-files-based-on-pattern-cut-can-occur-within-lines
# a generator for the file to be written to
def outfile_generator(filename):
n = -1
while True:
n += 1
yield open('{0}-{1}'.format(filename,n), 'w')
def split_file(filename):
"""
Split the input file into separate files, each containing a single patent.
As a hint - each patent declaration starts with the same line that was
causing the error found in the previous exercises.
The new files should be saved with filename in the following format:
"{}-{}".format(filename, n) where n is a counter, starting from 0.
"""
# the pattern on which the file is split
pat = '<?xml'
# an iterator for the filename
outfile_iterator = outfile_generator(filename)
with open(filename) as infile:
for line in infile:
# new file
if pat in line:
outfile = next(outfile_iterator)
# write the line
outfile.write(line)
def test():
split_file(PATENTS)
for n in range(4):
try:
fname = "{}-{}".format(PATENTS, n)
f = open(fname, "r")
if not f.readline().startswith("<?xml"):
print "You have not split the file {} in the correct boundary!".format(fname)
f.close()
except:
print "Could not find file {}. Check if the filename is correct!".format(fname)
#get_root(PATENTS)
test()