diff --git a/server/szurubooru/func/net.py b/server/szurubooru/func/net.py new file mode 100644 index 0000000..48a9352 --- /dev/null +++ b/server/szurubooru/func/net.py @@ -0,0 +1,7 @@ +import urllib.request + +def download(url): + request = urllib.request.Request(url) + request.add_header('Referer', url) + with urllib.request.urlopen(request) as handle: + return handle.read() diff --git a/server/szurubooru/tests/func/test_net.py b/server/szurubooru/tests/func/test_net.py new file mode 100644 index 0000000..4fd625d --- /dev/null +++ b/server/szurubooru/tests/func/test_net.py @@ -0,0 +1,45 @@ +from szurubooru.func import net + +def test_download(): + url = 'http://info.cern.ch/hypertext/WWW/TheProject.html' + + expected_content = ( + b'
\nThe World Wide Web project\n\n
\n\n

World Wide Web

The WorldWideWeb' + + b' (W3) is a wide-area\nhypermedia information retrieval\ninitiative aiming to give universal\na' + + b'ccess to a large universe of documents.

\nEverything there is ' + + b'online about\nW3 is linked directly or indirectly\nto this docum' + + b'ent, including an executive\nsum' + + b'mary of the project, Mailing lists\n, Policy , November\'s W3 news ,\nFrequently Ask' + + b'ed Questions .\n

\n
What\'s out there?\n
Pointers to the\nworld\'s ' + + b'online information, subjects\n, W3 servers, etc.\n
Help\n
on the browser you are using\n
Software Products\n
A list of W' + + b'3 project\ncomponents and their current state.\n(e.g. Line Mode ,X11 Viola , NeXTStep\n, Servers , Tools , Mail robot ,<' + + b'A\nNAME=52 HREF="Status.html#57">\nLibrary )\n
Technical\n
Details of protocols' + + b', formats,\nprogram internals etc\n
Bibliography\n
Paper documentation\non W3 a' + + b'nd references.\n
People\n<' + + b'DD> A list of some people involved\nin the project.\n
History\n
A summary of the hist' + + b'ory\nof the project.\n
How ca' + + b'n I help ?\n
If you would like\nto support the web..\nGetting code\n
Getti' + + b'ng the code by\nanonymous FTP , etc.\n
\n\n') + + actual_content = net.download(url) + assert actual_content == expected_content