diff options
author | volpino <fox91@anche.no> | 2012-10-30 20:27:37 +0100 |
---|---|---|
committer | volpino <fox91@anche.no> | 2012-10-30 20:27:37 +0100 |
commit | 48c1eadab4dd2132f777ff9b4b84fd35e46c9aee (patch) | |
tree | 58710752e7e7b350885431e7bf30003c2de31b11 | |
parent | euscanwww: Adding an about config page (diff) | |
parent | euscan/handlers: we need the file version (diff) | |
download | euscan-48c1eadab4dd2132f777ff9b4b84fd35e46c9aee.tar.gz euscan-48c1eadab4dd2132f777ff9b4b84fd35e46c9aee.tar.bz2 euscan-48c1eadab4dd2132f777ff9b4b84fd35e46c9aee.zip |
Merge branch 'master' of https://github.com/iksaif/euscan
Conflicts:
TODO
euscanwww/djeuscan/processing/scan/scan_portage.py
euscanwww/djeuscan/tasks.py
euscanwww/djeuscan/templates/euscan/_package_details.html
Signed-off-by: volpino <fox91@anche.no>
25 files changed, 167 insertions, 99 deletions
@@ -52,6 +52,9 @@ euscan euscanwww --------- +- Really fix mails +- Add an /about/config page that describe the current config (overlays, stuff in make.conf, euscan default settings, etc..) +- Create a rss for "scan world" + packages in GET - Always keep in db all found versions (when using an API only?). But don't display them if older than current packaged version, except maybe in the "upstream_version" column. ### packages @@ -292,9 +292,9 @@ def main(): from os.path import basename # To get the short name - output.error( + output.eerror( "The short ebuild name '%s' is ambiguous. Please specify" % - basename(pkgs[0]), + basename(pkgs[0]) + "one of the above fully-qualified ebuild names instead." ) exit_helper(1) diff --git a/euscanwww/djeuscan/api/handlers.py b/euscanwww/djeuscan/api/handlers.py index 18e1aca..922fe16 100644 --- a/euscanwww/djeuscan/api/handlers.py +++ b/euscanwww/djeuscan/api/handlers.py @@ -97,8 +97,12 @@ class PackagesHandler(AnonymousBaseHandler): herd = Herd.objects.get(herd=kwargs['herd']) packages = Package.objects.for_herd(herd, last_versions=True) data = {'herd': herd} - elif 'maintainer_id' in kwargs: - maintainer = Maintainer.objects.get(id=kwargs['maintainer_id']) + elif 'maintainer_id' in kwargs or 'maintainer_email' in kwargs: + if 'maintainer_id' in kwargs: + maintainer = Maintainer.objects.get(id=kwargs['maintainer_id']) + elif 'maintainer_email' in kwargs: + maintainer = Maintainer.objects.get(email=kwargs['maintainer_email']) + packages = Package.objects.for_maintainer( maintainer, last_versions=True diff --git a/euscanwww/djeuscan/api/urls.py b/euscanwww/djeuscan/api/urls.py index ffab482..578a8be 100644 --- a/euscanwww/djeuscan/api/urls.py +++ b/euscanwww/djeuscan/api/urls.py @@ -24,6 +24,8 @@ urlpatterns = patterns('djeuscan.api.views', url(r'^1.0/packages/by-maintainer/(?P<maintainer_id>\d+)\.(?P<emitter_format>.+)$', packages_handler, name='api.views.packages'), + url(r'^1.0/packages/by-maintainer/(?P<maintainer_email>[^/]+)\.(?P<emitter_format>.+)$', + packages_handler, name='api.views.packages'), url(r'^1.0/packages/by-herd/(?P<herd>[\@\{\}\w+.-]*)\.(?P<emitter_format>.+)?$', packages_handler, name='api.views.packages'), url(r'^1.0/packages/by-category/(?P<category>[\w+][\w+.-]*)\.(?P<emitter_format>.+)?$', diff --git a/euscanwww/djeuscan/feeds.py b/euscanwww/djeuscan/feeds.py index 7399618..98590a9 100644 --- a/euscanwww/djeuscan/feeds.py +++ b/euscanwww/djeuscan/feeds.py @@ -146,9 +146,16 @@ class PackageFeed(BaseFeed): class MaintainerFeed(BaseFeed): feed_type = Atom1Feed - def get_object(self, request, maintainer_id): + def get_object(self, request, maintainer_id=None, maintainer_email=None): + print maintainer_id, maintainer_email + if maintainer_id: + obj = get_object_or_404(Maintainer, id=maintainer_id) + else: + obj = get_object_or_404(Maintainer, email=maintainer_email) + + print obj return { - "obj": get_object_or_404(Maintainer, id=maintainer_id), + "obj": obj, "options": request.GET, } @@ -159,8 +166,7 @@ class MaintainerFeed(BaseFeed): return "Last changes for %s" % data["obj"] def link(self, data): - return reverse('djeuscan.views.maintainer', - kwargs={'maintainer_id': data["obj"].id}) + return reverse('djeuscan.views.maintainer', args=[data["obj"].email]) def _items(self, data): return VersionLog.objects.for_maintainer(data["obj"]), 50 diff --git a/euscanwww/djeuscan/processing/misc/update_counters.py b/euscanwww/djeuscan/processing/misc/update_counters.py index fdbea9c..392b7b5 100644 --- a/euscanwww/djeuscan/processing/misc/update_counters.py +++ b/euscanwww/djeuscan/processing/misc/update_counters.py @@ -114,7 +114,6 @@ def update_counters(fast=False, nolog=False, logger=None): package.last_version_upstream_id = last_versions_upstream.get( package.id, default )['id'] - package.save() n_packages_gentoo = int(package.n_packaged == package.n_versions) diff --git a/euscanwww/djeuscan/processing/scan/scan_portage.py b/euscanwww/djeuscan/processing/scan/scan_portage.py index 1ce7197..1d6c866 100644 --- a/euscanwww/djeuscan/processing/scan/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan/scan_portage.py @@ -13,7 +13,6 @@ from django.core.management.color import color_style from euscan.version import get_version_type from djeuscan.processing import FakeLogger -from djeuscan.processing.scan.scan_upstream import scan_upstream from djeuscan.models import Package, Version, VersionLog, Category, Overlay @@ -30,6 +29,10 @@ class ScanPortage(object): self._cache = {'packages': {}, 'versions': {}} self._overlays = None + self._updated_packages = set() + + def updated_packages(self): + return list(self._updated_packages) def cache_hash_package(self, category, name): return '%s/%s' % (category, name) @@ -200,13 +203,14 @@ class ScanPortage(object): packages_alive.add("%s/%s" % (cat, pkg)) new_version = False for cpv, slot, overlay, overlay_path in data['versions']: - new_version = new_version or self.store_version( + obj, created = self.store_version( package, cpv, slot, overlay, overlay_path ) + new_version = created or new_version # If the package has at least one new version scan upstream for it - if new_version and self.upstream: - scan_upstream([package], self.purge_versions, self.logger) + if new_version: + self._updated_packages.add(package) self.purge_old_packages(current_packages, packages_alive) self.purge_old_versions() @@ -269,7 +273,7 @@ class ScanPortage(object): # nothing to do (note: it can't be an upstream version because # overlay can't be empty here) if not created: - return False + return obj, created # New version created self.logger.info('+ [v] %s' % (obj)) @@ -292,7 +296,7 @@ class ScanPortage(object): vtype=obj.vtype, ) - return True + return obj, created def purge_old_packages(self, packages, alive): if not self.purge_packages: @@ -349,8 +353,7 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False, logger=logger, no_log=no_log, purge_packages=purge_packages, - purge_versions=purge_versions, - upstream=upstream + purge_versions=purge_versions ) logger.info('Scanning portage tree...') @@ -368,8 +371,10 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False, scan_handler.cache_store_version(version) logger.info('done') - if not packages: + if not packages and category: scan_handler.scan(category=category) + elif not packages: + scan_handler.scan() else: for pkg in packages: if isinstance(pkg, Package): @@ -394,3 +399,4 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False, logger.info("+ [o] %s", overlay["overlay"]) logger.info('Done.') + return scan_handler.updated_packages() diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py index d3c1b5b..769e4af 100644 --- a/euscanwww/djeuscan/tasks.py +++ b/euscanwww/djeuscan/tasks.py @@ -9,7 +9,6 @@ from celery.task import task, group #import portage from django.conf import settings -from django.core.cache import cache from django.template.loader import render_to_string from django.core.mail import send_mail from django.db.models import Q @@ -80,7 +79,6 @@ def update_counters(fast=False): """ Updates counters """ - logger = update_counters.get_logger() logger.info("Updating counters (fast=%s)...", fast) misc.update_counters(fast=fast) @@ -114,7 +112,7 @@ def scan_metadata(packages=[], category=None, populate=False): @task -def scan_portage(packages=[], category=None, upstream=True, +def scan_portage(packages=None, category=None, no_log=False, purge_packages=False, purge_versions=False, prefetch=False): """ @@ -131,17 +129,15 @@ def scan_portage(packages=[], category=None, upstream=True, else: logger.info("Starting portage scan...") - scan.scan_portage( + return scan.scan_portage( packages=packages, category=category, - upstream=upstream, no_log=no_log, purge_packages=purge_packages, purge_versions=purge_versions, prefetch=prefetch, logger=logger, ) - return True @task @@ -181,9 +177,17 @@ def update_portage(packages=None): # Workaround for celery bug when chaining groups update_portage_trees() - scan_portage(packages=[], purge_packages=True, purge_versions=True, - prefetch=True) - scan_metadata(packages=[], populate=True) + updated_packages = scan_portage( + packages=None, + purge_packages=True, + purge_versions=True, + prefetch=True + ) + scan_metadata(packages=None, populate=True) + if updated_packages: + group_chunks(scan_upstream, updated_packages, + settings.TASKS_UPSTREAM_GROUPS, + purge_versions=True)() update_counters(fast=False) """ Currently broken @@ -236,20 +240,12 @@ def scan_package_user(package): return True -@task +@task(rate_limit="1/m") def consume_refresh_queue(locked=False): """ Satisfies user requests for package refreshing, runs every minute """ - LOCK_ID = 'lock-consume-refresh-queue' - unlock = lambda: cache.delete(LOCK_ID) - lock = lambda: cache.add(LOCK_ID, True, 120) - logger = consume_refresh_queue.get_logger() - - if not locked and not lock(): - return - logger.info('Consuming package refresh request queue...') try: @@ -259,16 +255,14 @@ def consume_refresh_queue(locked=False): scan_package_user.delay(pkg) logger.info('Selected: %s' % pkg) except IndexError: - pass - finally: - unlock() + return if RefreshPackageQuery.objects.count(): logger.info('Restarting myself in 60s') - lock() consume_refresh_queue.apply_async( kwargs={'locked': True}, countdown=60 ) + return True @task(max_retries=10, default_retry_delay=10 * 60) @@ -280,6 +274,7 @@ def send_user_email(address, subject, text): ) except Exception, exc: raise send_user_email.retry(exc=exc) + return True @task @@ -322,6 +317,7 @@ def process_emails(profiles, only_if_vlogs=False): profile.last_email = now profile.save(force_update=True) + return True @task @@ -336,6 +332,7 @@ def send_update_email(): settings.TASKS_EMAIL_GROUPS, only_if_vlogs=True )() + return True @task @@ -345,6 +342,7 @@ def send_weekly_email(): email_activated=True ) group_chunks(process_emails, profiles, settings.TASKS_EMAIL_GROUPS)() + return True @task @@ -354,7 +352,7 @@ def send_monthly_email(): email_activated=True ) group_chunks(process_emails, profiles, settings.TASKS_EMAIL_GROUPS)() - + return True admin_tasks = [ regen_rrds, diff --git a/euscanwww/djeuscan/templates/euscan/_maintainers_table.html b/euscanwww/djeuscan/templates/euscan/_maintainers_table.html index bfbdd21..665e3e8 100644 --- a/euscanwww/djeuscan/templates/euscan/_maintainers_table.html +++ b/euscanwww/djeuscan/templates/euscan/_maintainers_table.html @@ -18,7 +18,7 @@ {% for maintainer in maintainers %} <tr> <td> - <a href="{% url "maintainer" maintainer.maintainers__id %}"> + <a href="{% url "maintainer" maintainer.maintainers__email %}"> {% if maintainer.maintainers__name != maintainer.maintainers__email %} {{ maintainer.maintainers__name }} <{{ maintainer.maintainers__email }}> {% else %} @@ -31,8 +31,8 @@ {% if extras %} <td> - <img src="{% url "chart_maintainer" maintainer.maintainers__id 'packages-monthly-small' %}" /> - <img src="{% url "chart_maintainer" maintainer.maintainers__id 'versions-monthly-small' %}" /> + <img src="{% url "chart_maintainer" maintainer.maintainers__email 'packages-monthly-small' %}" /> + <img src="{% url "chart_maintainer" maintainer.maintainers__email 'versions-monthly-small' %}" /> </td> {% endif %} diff --git a/euscanwww/djeuscan/templates/euscan/_package_details.html b/euscanwww/djeuscan/templates/euscan/_package_details.html index 8746088..7f92133 100644 --- a/euscanwww/djeuscan/templates/euscan/_package_details.html +++ b/euscanwww/djeuscan/templates/euscan/_package_details.html @@ -33,12 +33,12 @@ {% for maintainer in package.maintainers.all %} <p> {% if maintainer.name != maintainer.email %} - <a href="{% url "maintainer" maintainer.id %}"> + <a href="{% url "maintainer" maintainer.email %}"> {{ maintainer.name }} </a> <{{ maintainer.email }}> {% else %} - <a href="{% url "maintainer" maintainer.id %}"> + <a href="{% url "maintainer" maintainer.email %}"> {{ maintainer.email }} </a> {% endif %} diff --git a/euscanwww/djeuscan/templates/euscan/accounts/index.html b/euscanwww/djeuscan/templates/euscan/accounts/index.html index e9ac816..0e61ed6 100644 --- a/euscanwww/djeuscan/templates/euscan/accounts/index.html +++ b/euscanwww/djeuscan/templates/euscan/accounts/index.html @@ -75,7 +75,7 @@ Hot maintainers: <ul> {% for c in maintainers|slice:":3" %} - <li><a href="{% url "maintainer" c.maintainers__id %}">{{ c.maintainers__name }}</a> ({{ c.n_versions|sub:c.n_packaged|sub:c.n_overlay }})</li> + <li><a href="{% url "maintainer" c.maintainers__email %}">{{ c.maintainers__name }}</a> ({{ c.n_versions|sub:c.n_packaged|sub:c.n_overlay }})</li> {% endfor %} </ul> </li> diff --git a/euscanwww/djeuscan/templates/euscan/maintainer.html b/euscanwww/djeuscan/templates/euscan/maintainer.html index 65f498c..d73b582 100644 --- a/euscanwww/djeuscan/templates/euscan/maintainer.html +++ b/euscanwww/djeuscan/templates/euscan/maintainer.html @@ -11,7 +11,7 @@ {{ block.super }} <li> <img src="{{ STATIC_URL }}/img/feed.png" alt="feed" /> - <a title="{{ maintainer.name }} Feed" href="{% url "maintainer_feed" maintainer.id %}"> + <a title="{{ maintainer.name }} Feed" href="{% url "maintainer_feed" maintainer.email %}"> {{ maintainer.name|truncatechars:15 }} </a> </li> @@ -22,10 +22,10 @@ Maintainer: {{ maintainer.name }} <{{ maintainer.email }}> {% if user.is_authenticated %} <span class="pull-right"> - <button class="btn favourite-button {% if favourited %}hide{% endif %}" data-url="{% url "favourite_maintainer" maintainer.id %}"> + <button class="btn favourite-button {% if favourited %}hide{% endif %}" data-url="{% url "favourite_maintainer" maintainer.email %}"> <img src="{{ STATIC_URL}}/img/watch-icon.png" alt="Watch" /> </button> - <button class="btn unfavourite-button {% if not favourited %}hide{% endif %}" data-url="{% url "unfavourite_maintainer" maintainer.id %}"> + <button class="btn unfavourite-button {% if not favourited %}hide{% endif %}" data-url="{% url "unfavourite_maintainer" maintainer.email %}"> <img src="{{ STATIC_URL}}/img/unwatch-icon.png" alt="Unwatch" /> </button> </span> @@ -37,19 +37,19 @@ <h3>Statistics</h3> <h4>Current statistics</h4> <p> - <img alt="pie versions" src="{% url "chart_maintainer" maintainer.id 'pie-versions' %}" /> - <img alt="pie packages" src="{% url "chart_maintainer" maintainer.id 'pie-packages' %}" /> + <img alt="pie versions" src="{% url "chart_maintainer" maintainer.email 'pie-versions' %}" /> + <img alt="pie packages" src="{% url "chart_maintainer" maintainer.email 'pie-packages' %}" /> </p> <h4>All Time</h4> <p> - <img alt="version weekly" src="{% url "chart_maintainer" maintainer.id 'versions-weekly' %}" /> - <img alt="versions monthly" src="{% url "chart_maintainer" maintainer.id 'versions-monthly' %}" /> + <img alt="version weekly" src="{% url "chart_maintainer" maintainer.email 'versions-weekly' %}" /> + <img alt="versions monthly" src="{% url "chart_maintainer" maintainer.email 'versions-monthly' %}" /> </p> <p> - <img alt="packages weekly" src="{% url "chart_maintainer" maintainer.id 'packages-weekly' %}" /> - <img alt="packages monthly" src="{% url "chart_maintainer" maintainer.id 'packages-monthly' %}" /> + <img alt="packages weekly" src="{% url "chart_maintainer" maintainer.email 'packages-weekly' %}" /> + <img alt="packages monthly" src="{% url "chart_maintainer" maintainer.email 'packages-monthly' %}" /> </p> <script type="text/javascript"> @@ -77,9 +77,9 @@ $(document).ready(function () { <div class="menu rounded notfirst-menu"> <dl> <dt>Versions</dt> - <dd><img alt="versions" src="{% url "chart_maintainer" maintainer.id 'versions-monthly-small' %}" /></dd> + <dd><img alt="versions" src="{% url "chart_maintainer" maintainer.email 'versions-monthly-small' %}" /></dd> <dt>Packages</dt> - <dd><img alt="packages" src="{% url "chart_maintainer" maintainer.id 'packages-monthly-small' %}" /></dd> + <dd><img alt="packages" src="{% url "chart_maintainer" maintainer.email 'packages-monthly-small' %}" /></dd> </dl> </div> diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py index 789fc9f..1d70292 100644 --- a/euscanwww/djeuscan/urls.py +++ b/euscanwww/djeuscan/urls.py @@ -55,16 +55,13 @@ herds_patterns = patterns('djeuscan.views', url(r'^$', 'herds', name="herds"), ) +prefix = '^((?P<maintainer_id>\d+)|(?P<maintainer_email>[^/]+))' maintainers_patterns = patterns('djeuscan.views', - url(r'^(?P<maintainer_id>\d+)/(view/)?$', 'maintainer', name="maintainer"), - url(r'^(?P<maintainer_id>\d+)/feed/$', MaintainerFeed(), - name='maintainer_feed'), - url(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$', - 'chart_maintainer', name="chart_maintainer"), - url(r'^(?P<maintainer_id>\d+)/favourite/$', - 'favourite_maintainer', name="favourite_maintainer"), - url(r'^(?P<maintainer_id>\d+)/unfavourite/$', - 'unfavourite_maintainer', name="unfavourite_maintainer"), + url(prefix + r'/(view/)?$', 'maintainer', name="maintainer"), + url(prefix + r'/feed/$', MaintainerFeed(), name='maintainer_feed'), + url(prefix + r'/charts/(?P<chart>[\w\-]+).png$', 'chart_maintainer', name="chart_maintainer"), + url(prefix + r'/favourite/$', 'favourite_maintainer', name="favourite_maintainer"), + url(prefix + r'/unfavourite/$', 'unfavourite_maintainer', name="unfavourite_maintainer"), url(r'^$', 'maintainers', name="maintainers"), ) diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index cc0d2f2..5f16a43 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -122,9 +122,15 @@ def maintainers(request): return {'maintainers': maintainers, 'last_scan': last_scan} +def get_maintainer_or_404(id=None, email=None): + if id: + return get_object_or_404(Maintainer, pk=id) + else: + return get_object_or_404(Maintainer, email=email) + @render_to('euscan/maintainer.html') -def maintainer(request, maintainer_id): - maintainer = get_object_or_404(Maintainer, pk=maintainer_id) +def maintainer(request, maintainer_id=None, maintainer_email=None): + maintainer = get_maintainer_or_404(maintainer_id, maintainer_email) packages = Package.objects.for_maintainer(maintainer, last_versions=True) try: @@ -337,10 +343,10 @@ def chart(request, **kwargs): chart = kwargs['chart'] if 'chart' in kwargs else None - if 'maintainer_id' in kwargs: - kwargs['maintainer'] = get_object_or_404( - Maintainer, - id=kwargs['maintainer_id'] + if 'maintainer_id' in kwargs or 'maintainer_email' in kwargs: + kwargs['maintainer'] = get_maintainer_or_404( + kwargs.get('maintainer_id'), + kwargs.get('maintainer_email') ) if 'herd' in kwargs: kwargs['herd'] = get_object_or_404(Herd, herd=kwargs['herd']) @@ -550,8 +556,8 @@ def unfavourite_herd(request, herd): @login_required @require_POST @ajax_request -def favourite_maintainer(request, maintainer_id): - obj = get_object_or_404(Maintainer, pk=maintainer_id) +def favourite_maintainer(request, maintainer_id=None, maintainer_email=None): + obj = get_maintainer_or_404(maintainer_id, maintainer_email) get_profile(request.user).maintainers.add(obj) return {"success": True} @@ -559,8 +565,8 @@ def favourite_maintainer(request, maintainer_id): @login_required @require_POST @ajax_request -def unfavourite_maintainer(request, maintainer_id): - maintainer = get_object_or_404(Maintainer, pk=maintainer_id) +def unfavourite_maintainer(request, maintainer_id=None, maintainer_email=None): + obj = get_maintainer_or_404(maintainer_id, maintainer_email) get_profile(request.user).maintainers.remove(maintainer) return {"success": True} diff --git a/euscanwww/euscanwww/captcha/__init__.py b/euscanwww/euscanwww/captcha/__init__.py new file mode 100644 index 0000000..a6b282a --- /dev/null +++ b/euscanwww/euscanwww/captcha/__init__.py @@ -0,0 +1,6 @@ +from registration.backends.default import DefaultBackend +from forms import RegistrationFormCaptcha + +class CaptchaDefaultBackend(DefaultBackend): + def get_form_class(self, request): + return RegistrationFormCaptcha diff --git a/euscanwww/euscanwww/captcha/forms.py b/euscanwww/euscanwww/captcha/forms.py new file mode 100644 index 0000000..ca4a876 --- /dev/null +++ b/euscanwww/euscanwww/captcha/forms.py @@ -0,0 +1,5 @@ +from captcha.fields import ReCaptchaField +from registration.forms import RegistrationForm + +class RegistrationFormCaptcha(RegistrationForm): + captcha = ReCaptchaField(attrs={'theme': 'white'}) diff --git a/euscanwww/euscanwww/captcha/urls.py b/euscanwww/euscanwww/captcha/urls.py new file mode 100644 index 0000000..76cbd2e --- /dev/null +++ b/euscanwww/euscanwww/captcha/urls.py @@ -0,0 +1,12 @@ +from django.conf.urls.defaults import patterns, url + +from registration.views import register + +urlpatterns = patterns('', + url( + r'^register/$', + register, + {'backend': 'euscanwww.captcha.CaptchaDefaultBackend'}, + name='registration_register' + ), +) diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py index d6b8616..5705c3b 100644 --- a/euscanwww/euscanwww/settings.py +++ b/euscanwww/euscanwww/settings.py @@ -267,3 +267,4 @@ except ImportError, ex: os.environ['ROOT'] = PORTAGE_ROOT os.environ['PORTAGE_CONFIGROOT'] = PORTAGE_CONFIGROOT os.environ['EIX_CACHEFILE'] = EIX_CACHEFILE +os.environ['http_proxy'] = '127.0.0.1:8123' diff --git a/euscanwww/euscanwww/urls.py b/euscanwww/euscanwww/urls.py index d6090b5..b104339 100644 --- a/euscanwww/euscanwww/urls.py +++ b/euscanwww/euscanwww/urls.py @@ -8,7 +8,7 @@ urlpatterns = patterns('', url(r'^', include('djeuscan.urls')), url(r'^admin/', include(admin.site.urls)), - url(r'^accounts/', include('captcha.backends.default.urls')), + url(r'^accounts/', include('euscanwww.captcha.urls')), url(r'^accounts/', include('registration.backends.default.urls')), ) diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py index 98daeee..49241dd 100644 --- a/pym/euscan/__init__.py +++ b/pym/euscan/__init__.py @@ -79,11 +79,12 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [ '(.*)sourceforge(.*)', '(.*)github.com', '(.*)berlios(.*)', - '(.*)qt.nokia.com(.*)', - '(.*)chromium.org(.*)', - '(.*)nodejs.org(.*)', - '(.*)download.mono-project.com(.*)', - '(.*)fedorahosted.org(.*)', + '(.*)qt\.nokia\.com(.*)', + '(.*)chromium\.org(.*)', + '(.*)nodejs\.org(.*)', + '(.*)download\.mono-project\.com(.*)', + '(.*)fedorahosted\.org(.*)', + '(.*)download\.tuxfamily\.org(.*)', ] from out import EuscanOutput diff --git a/pym/euscan/handlers/berlios.py b/pym/euscan/handlers/berlios.py index bbfe6e1..11471e4 100644 --- a/pym/euscan/handlers/berlios.py +++ b/pym/euscan/handlers/berlios.py @@ -16,7 +16,14 @@ berlios_regex = r"mirror://berlios/([^/]+)/([^/]+)" def can_handle(pkg, url=None): - return url and re.search(berlios_regex, url) + if not url: + return False + + cp, ver, rev = portage.pkgsplit(pkg.cpv) + if ver not in url: + return False + + return re.search(berlios_regex, url) def scan_url(pkg, url, options): diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index fd82c71..0795488 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -70,6 +70,7 @@ def scan_html(data, url, pattern): match.group(0)) ) + return results diff --git a/pym/euscan/handlers/google_code.py b/pym/euscan/handlers/google_code.py index 892064f..d6c5bd1 100644 --- a/pym/euscan/handlers/google_code.py +++ b/pym/euscan/handlers/google_code.py @@ -14,8 +14,14 @@ package_name_regex = r"http://(.+).googlecode.com/files/.+" def can_handle(pkg, url=None): - return url and re.match(package_name_regex, url) + if not url: + return False + cp, ver, rev = portage.pkgsplit(pkg.cpv) + if ver not in url: + return False + + return re.match(package_name_regex, url) def scan_url(pkg, url, options): output.einfo("Using Google Code handler") diff --git a/pym/euscan/handlers/sourceforge.py b/pym/euscan/handlers/sourceforge.py index 11563e0..0508289 100644 --- a/pym/euscan/handlers/sourceforge.py +++ b/pym/euscan/handlers/sourceforge.py @@ -12,7 +12,14 @@ PRIORITY = 90 def can_handle(pkg, url=None): - return url and "mirror://sourceforge/" in url + if not url: + return False + + cp, ver, rev = portage.pkgsplit(pkg.cpv) + if ver not in url: + return False + + return "mirror://sourceforge/" in url def scan_url(pkg, url, options): diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index 5e3e6ea..3271811 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -373,27 +373,28 @@ def tryurl(fileurl, template): def regex_from_template(template): # Escape - template = re.escape(template) + regexp = re.escape(template) # Unescape specific stuff - template = template.replace('\$\{', '${') - template = template.replace('\}', '}') - template = template.replace('}\.$', '}.$') + regexp = regexp.replace('\$\{', '${') + regexp = regexp.replace('\}', '}') + regexp = regexp.replace('}\.$', '}.$') # Replace ${\d+} - #template = template.replace('${0}', r'([\d]+?)') - template = re.sub(r'(\$\{\d+\}(\.?))+', r'([\w\.]+?)', template) + #regexp = regexp.replace('${0}', r'([\d]+?)') + regexp = re.sub(r'(\$\{\d+\}(\.?))+', r'([\w\.]+?)', regexp) - #template = re.sub(r'(\$\{\d+\}\.?)+', r'([\w]+?)', template) - #template = re.sub(r'(\$\{\d+\}\.+)+', '(.+?)\.', template) - #template = re.sub(r'(\$\{\d+\})+', '(.+?)', template) + #regexp = re.sub(r'(\$\{\d+\}\.?)+', r'([\w]+?)', regexp) + #regexp = re.sub(r'(\$\{\d+\}\.+)+', '(.+?)\.', regexp) + #regexp = re.sub(r'(\$\{\d+\})+', '(.+?)', regexp) # Full version - template = template.replace('${PV}', _v) + regexp = regexp.replace('${PV}', _v) # End - template = template + r'/?$' - return template + regexp = regexp + r'/?$' + + return regexp def basedir_from_template(template): @@ -417,7 +418,7 @@ def generate_scan_paths(url): path = prefix + ":/" for chunk in chunks: if '${' in chunk: - steps.append((path, regex_from_template(chunk))) + steps.append((path, '^(?:|.*/)' + regex_from_template(chunk))) path = "" else: path += "/" |