<html><head><meta name="color-scheme" content="light dark"></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">#!/usr/bin/env perl

#
# unique-domain-builder.pl
#
#   (C) 2010 JosÃ© Miguel Parrella Romero
#   This is free software, distributed under the same terms of Perl.
#

use strict;

use Yahoo::Search;
use Domain::PublicSuffix;

my $n;       # Total number of results
my $i = 0;   # Current search result
my $c = 100; # Maximum search results size

my $cctld   = $ARGV[0] || 've';
my $query   = "site:gob.$cctld OR site:gov.$cctld";
my %options = ( AppId =&gt; 'UDB 0.1',
                Type =&gt; 'any', AllowSimilar =&gt; 0, Language =&gt; undef );

# Optional list of prefixes for the base domain results
my @prefixes = qw/ipv6 6 www6 w6 www.v6 www.ipv6/;

# Get the search results count
my $search = Yahoo::Search-&gt;new();
my $req    = $search-&gt;Request( Doc =&gt; $query, %options, Start =&gt; 0, Count =&gt; 1 );
my $res    = $req-&gt;Fetch();
   $n      = $res-&gt;CountAvail;

# Initialize D::PS
my $d = new Domain::PublicSuffix({'data_file'=&gt;'./suffices'});

my %domains; # List of unique domains

while ( $i &lt; $n ) {
  # Build an iterative query
  my @s = Yahoo::Search-&gt;Results(
	Doc   =&gt; $query,
	Start =&gt; $i,
	Count =&gt; $c,
	%options
  );
  exit $@ if $@; # we exit if we have an error due to Yahoo! returning 400 when limit is reached

  my %f; # list of unique domains per-iteration

  foreach (@s) { # a search result
	++$i;
	my $u = $_-&gt;Url;
           $u =~ s#^http://##; # stripping stuff...
	   $u =~ s#/.+##;      # then some more...
	   $u =~ s#/##;        # and... done!

	my $b = $d-&gt;get_root_domain($u); # D::PS does its work

	unless ($domains{$b}) { # domain isn't registered
		$domains{$b} = $_-&gt;Title; # store it in the global list
		$f{$b} = $_-&gt;Title;       # store it in the session list
	}
  }

  foreach my $e ( keys %f ) { # session list of unique domains
	print "$e\n";
	foreach my $p ( @prefixes ) { # prefixes
		print "$p.$e\n";
	}
  }
}
</pre></body></html>