diff --git a/README.md b/README.md index 67682a9..556d556 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ Pytor ===== +PyTor +python wrapper for anonymizing (web scraping?) requests using Tor. + +Intially forked from https://github.com/bdheath/pytor (not properly written) + +But I am **rewriting** the entire thing with some extensions (see #TODOs) + +### Need to update this readme + + Pytor is a Python wrapper for scraping over the Tor network. This is useful if you've been blocked (either locally or remotely) from the server you're attempting to scrape, or if it's otherwise important to not reveal your identity. @@ -7,8 +17,7 @@ Pytor allows you to channel simple http requests through a Tor proxy. It also al Requirements ============ -* A functioning (and running) instance of Tor. This could be the basic Tor Browser Bundle. For more configuration options, consider using the Tor Expert Bundle, available from https://www.torproject.org/download/download.html.en. -* Mechanize (install through pip) +* A functioning (and running) instance of Tor. This could be the basic Tor Browser Bundle. For more configuration options, consider using the Tor Expert Bundle, available from https://www.torproject.org/download/download.html.en. * Stem (install through pip or apt-get) Basic usage @@ -23,7 +32,7 @@ html = tor.get('http://bradheath.org') Or, if your Tor configuration requires a different host or port: ```python from pytor import pytor -tor = pytor(host='localhost', port=9055) +tor = pytor(host='localhost', socks_port=9055) html = tor.get('http://bradheath.org') ``` @@ -44,7 +53,7 @@ tor = pytor( port = 9050, controller = True, controlPort = 9051, password='Your- Request a new identity from Tor: (Note that the network won't always assign you one, and even when it does, you may end up with the same exit node and therefore the same IP address. Also note that you shouldn't change your identity too ofen to avoid stressing the network.) ```python -tor.newIdentity() +tor.getNewIdentity() ``` Have Pytor periodically assign a new identity. (Note that this currently works with get() and download_file() requests, but does not currently force a new identity when using a mechanize browser. More on that later.) @@ -52,6 +61,8 @@ Have Pytor periodically assign a new identity. (Note that this currently works w tor.identityTime(1200) # Request a new identity every 1,200 seconds (20 minutes) ``` +deprecated: + To create a new instance of a mechanize browser: ```python br = tor.mechanizeBrowser() diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/__pycache__/pytor.cpython-35.pyc b/__pycache__/pytor.cpython-35.pyc new file mode 100644 index 0000000..69e787e Binary files /dev/null and b/__pycache__/pytor.cpython-35.pyc differ diff --git a/pytor-test.py b/pytor-test.py deleted file mode 100644 index fdad41c..0000000 --- a/pytor-test.py +++ /dev/null @@ -1,11 +0,0 @@ -from pytor import pytor -import urllib2 - -ip = urllib2.urlopen('http://bradheath.org/ip').read() -tor = pytor(controller=True, controlPort = 9051, port = 9060, password='YOURPASSWORD') - -print "Without Tor, my IP address is : " + ip -print "With Tor, my IP address is : " + tor.ip() -tor.newIdentity() -print "Now my IP address is : " + tor.ip() - diff --git a/pytor.py b/pytor.py index d1f64c3..5a20412 100644 --- a/pytor.py +++ b/pytor.py @@ -1,118 +1,63 @@ -# PYTOR -# Lightweight wrapper for web scraping on the Tor network -# Brad Heath, brad.heath@gmail.com, @bradheath -# -# REQUIREMENTS -# - Functioning (and active) TOR proxy -# - SocksiPy (socks.py installed to lib/site-packages) -# - Stem (pip install stem) -# - Mechanize -# -# +""" +PyTor +python wrapper for anonymizing (web scraping?) requests using Tor. + +Intially forked from https://github.com/bdheath/pytor +But I am rewriting the entire thing for my purpose + +""" -import stem from stem.control import Controller from stem import Signal -import mechanize -import datetime import requests -class pytor: +from datetime import datetime + + +#TODO - some sites block tor exit nodes - add user agent randomizer etc to fix this +#TODO - renew id interval (multi threading) +#TODO - get post : pass along the kwargs + +class PyTor: + def __init__(self, host='localhost', socks_port=9050, need_control=False, control_port=9051, password=''): - controlPort = 9051 - port = 9050 - host = 'localhost' - password = '' - _version = 0.1 - _last_result = None - _last_request = None - _id_time = None - _last_id_time = None - _ip = None - _connected = False - browser = None - controller = False - - _proxies = {} - def __init__(self, host='localhost', port=9050, controller=False, controlPort = 9051, password=''): - self.controlPort = controlPort - self.port = port - self.controller = controller - self._proxies['http'] = 'socks5://localhost:' + str(self.port) - self._proxies['https'] = 'socks5://localhost:' + str(self.port) - if self.controller: - self.torControl = Controller.from_port(port=controlPort) - self.torControl.authenticate(self.password) - self._connected = True - self._last_id_time = datetime.datetime.now() - return - - def request(self, url, timeout = 30): - self._checkIdentityTime() - r = requests.get(url, timeout=timeout, proxies=self._proxies) - self._last_request = url - self._last_result = r.text - return r - - def get(self, url, t=30): - self._checkIdentityTime() + self.host = host + self.socks_port = socks_port + self.need_control = need_control + self.control_port = control_port + self.password=password + self.proxies = {} + self.proxies['http'] = 'socks5://' + str(self.host) + ':' + str(self.socks_port) + self.proxies['https'] = 'socks5://' + str(self.host) + ':' + str(self.socks_port) - r = self.request(url, timeout = t) - - if r.status_code == 200: - self._last_result = r.text - self._last_request = url - return self._last_result - else: - return False + if self.need_control: + self.controller = Controller.from_port(address=self.host, port=control_port) + self.controller.authenticate(self.password) + else: + self.controller = None + return - def ip(self): - self._ip = self.get('http://bradheath.org/ip') - return self._ip - def timeForNew(self): - self._checkIdentityTime() - return - - def saveLastResult(file): - with open(file, 'wb') as local_file: - local_file.write(self._last_result) - return - - def downloadFile(self, url, file): - self._checkIdentityTime() - r = requests.get(url, stream = True, proxies = self._proxies) - with open(file, 'wb') as local_file: - for chunk in r.iter_content ( chunk_size = 1024) - if chunk: - local_file.write(chunk) - self._last_request = url - return True + def get(self, url, timeout=30): + response = requests.get(url, timeout=timeout, proxies=self.proxies) + return response - def newIdentity(self): - print "## NEW IDENTITY ## " - if not self._connected: - self.torControl = Controller.from_port(port=self.controlPort) - self.torControl.authenticate(self.password) - self._connected = True - self.torControl.authenticate(self.password) - self.torControl.signal(Signal.NEWNYM) - self._last_id_time = datetime.datetime.now() - return True + def post(self): + #TODO + pass - def _checkIdentityTime(self): - if self._id_time != None and self._last_id_time != None: - if (datetime.datetime.now() - self._last_id_time).seconds >= self._id_time: - print 'Getting new identity' - self.newIdentity() - return - - def identityTime(self, time = 600): - self._id_time = time - return - - def mechanizeBrowser(self): - self.browser = mechanize.Browser() - return self.browser + def ip(self): + response = self.get('http://icanhazip.com/') # standard url + if response.status_code==200: + self._ip=response.text + return self._ip + else: + return None + def get_new_identity(self): + if not self.controller: + self.controller = Controller.from_port(address=self.host, port=self.control_port) + self.controller.authenticate(self.password) + self.controller.signal(Signal.NEWNYM) + return self.ip() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9f5640f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +stem +requests +requests[socks] diff --git a/test/ip.py b/test/ip.py new file mode 100644 index 0000000..ba10ba3 --- /dev/null +++ b/test/ip.py @@ -0,0 +1,12 @@ +import requests + +tor_proxy = 'socks5://10.130.4.162:9050' #lab machine +socks_proxy = { 'http': tor_proxy, + 'https': tor_proxy + } + +current_ip = requests.get(url='http://icanhazip.com/',proxies=socks_proxy,verify=False) + +print(current_ip.text) + + diff --git a/test/new-ip.py b/test/new-ip.py new file mode 100644 index 0000000..3c3db0b --- /dev/null +++ b/test/new-ip.py @@ -0,0 +1,11 @@ +from stem import Signal +from stem.control import Controller + + +def get_new_ip(): + """Change IP using TOR""" + with Controller.from_port(address = '10.130.4.162', port=9051) as controller: + controller.authenticate(password='password') + controller.signal(Signal.NEWNYM) + +get_new_ip() diff --git a/test/privoxy.py b/test/privoxy.py new file mode 100644 index 0000000..a75a706 --- /dev/null +++ b/test/privoxy.py @@ -0,0 +1,15 @@ +import requests + +local_proxy = '10.130.4.162:8118' +http_proxy = { + 'http': local_proxy, + 'https': local_proxy +} + +current_ip = requests.get( + url='http://icanhazip.com/', + proxies=http_proxy, + verify=False +) + +print(current_ip.text) diff --git a/test/pytor-test.py b/test/pytor-test.py new file mode 100644 index 0000000..cea82e5 --- /dev/null +++ b/test/pytor-test.py @@ -0,0 +1,23 @@ +import sys +sys.path.append('../') + +from pytor import PyTor +import requests + +r = requests.get(url='http://icanhazip.com/') +if (r.status_code == 200): + ip = r.text + +tor = PyTor(host='10.130.4.162', need_control=True, control_port=9051, socks_port=9050, password='password') + +print("Without Tor, my IP address is : " + str(ip)) +print("With Tor, my IP address is : " + tor.ip()) +print("Getting new Ip from Tor #####") +tor.get_new_identity() +print("Now my IP address is : " + tor.ip()) + + +r=tor.get(url="http://www.thehindu.com/") + +if(r.status_code==200): + print(r.text) \ No newline at end of file