From b73646a8c3383302a3cd49a1de21c6bb2d476b43 Mon Sep 17 00:00:00 2001
From: Jesus Lanchas <jlanchas@acceso.com>
Date: Wed, 29 Feb 2012 14:18:54 +0100
Subject: [PATCH 1/2] Automatic detection of the char encoding using the
 juniversalchardet library. Moreover, the system now is prepared to use a
 language detection previously to count the stop words in each fragment.

---
 .../resources/com/gravity/goose/text/all.txt  | 1549 +++++++++++++++++
 .../resources/com/gravity/goose/text/ca.txt   |  219 +++
 .../resources/com/gravity/goose/text/de.txt   |  231 +++
 .../resources/com/gravity/goose/text/en.txt   |  546 ++++++
 .../resources/com/gravity/goose/text/es.txt   |  308 ++++
 .../resources/com/gravity/goose/text/fr.txt   |  163 ++
 .../resources/com/gravity/goose/text/it.txt   |  279 +++
 .../resources/com/gravity/goose/text/pt.txt   |  203 +++
 8 files changed, 3498 insertions(+)
 create mode 100644 src/main/resources/com/gravity/goose/text/all.txt
 create mode 100644 src/main/resources/com/gravity/goose/text/ca.txt
 create mode 100644 src/main/resources/com/gravity/goose/text/de.txt
 create mode 100644 src/main/resources/com/gravity/goose/text/en.txt
 create mode 100644 src/main/resources/com/gravity/goose/text/es.txt
 create mode 100644 src/main/resources/com/gravity/goose/text/fr.txt
 create mode 100644 src/main/resources/com/gravity/goose/text/it.txt
 create mode 100644 src/main/resources/com/gravity/goose/text/pt.txt

diff --git a/src/main/resources/com/gravity/goose/text/all.txt b/src/main/resources/com/gravity/goose/text/all.txt
new file mode 100644
index 000000000..c7f995fa5
--- /dev/null
+++ b/src/main/resources/com/gravity/goose/text/all.txt
@@ -0,0 +1,1549 @@
+a
+è
+über
+êtes
+abans
+abbia
+abbiamo
+abbiano
+abbiate
+aber
+about
+above
+ací
+across
+ad
+after
+afterwards
+again
+against
+agl
+agli
+ah
+ai
+aie
+aient
+aies
+ait
+així
+això
+al
+él
+aleshores
+algo
+algun
+alguna
+algunas
+algunes
+algunos
+alguns
+alhora
+all
+alla
+allí
+allò
+allà
+alle
+allem
+allen
+aller
+alles
+allo
+almost
+alone
+along
+already
+als
+also
+although
+altra
+altre
+altres
+always
+am
+amb
+ambdós
+ambdues
+among
+amongst
+amoungst
+amount
+an
+anche
+and
+ander
+andere
+anderem
+anderen
+anderer
+anderes
+anderm
+andern
+anderr
+anders
+another
+ante
+antes
+any
+anyhow
+anyone
+anything
+anyway
+anywhere
+ao
+aos
+apa
+aquí
+aquela
+aquelas
+aquele
+aqueles
+aquell
+aquella
+aquelles
+aquells
+aquest
+aquesta
+aquestes
+aquests
+aquilo
+éramos
+are
+érem
+éreu
+around
+as
+és
+at
+até
+été
+étée
+étées
+étaient
+étais
+était
+étant
+étés
+étiez
+étions
+au
+auch
+auf
+aura
+aurai
+auraient
+aurais
+aurait
+auras
+aurez
+auriez
+aurions
+aurons
+auront
+aus
+aux
+avaient
+avais
+avait
+avec
+avemmo
+avendo
+avesse
+avessero
+avessi
+avessimo
+aveste
+avesti
+avete
+aveva
+avevamo
+avevano
+avevate
+avevi
+avevo
+avez
+aviez
+avions
+avons
+avrò
+avrai
+avranno
+avrà
+avrebbe
+avrebbero
+avrei
+avremmo
+avremo
+avreste
+avresti
+avrete
+avuta
+avute
+avuti
+avuto
+ayant
+ayez
+ayons
+à
+às
+back
+baix
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+behind
+bei
+being
+below
+beside
+besides
+between
+beyond
+bill
+bin
+bis
+bist
+both
+bottom
+but
+by
+c
+cada
+cadascú
+cadascuna
+cadascunes
+cadascuns
+call
+can
+cannot
+cant
+ce
+ceci
+celà
+ces
+cet
+cette
+che
+chi
+ci
+co
+coi
+col
+com
+come
+como
+computer
+con
+contra
+contro
+could
+couldnt
+cry
+cual
+cuando
+cui
+d
+da
+daß
+dagl
+dagli
+dai
+dal
+dall
+dalla
+dalle
+dallo
+dalt
+damit
+dann
+dans
+das
+dasselbe
+dazu
+de
+degl
+degli
+dei
+dein
+deine
+deinem
+deinen
+deiner
+deines
+del
+dela
+delas
+dele
+deles
+dell
+della
+delle
+dello
+dels
+dem
+demselben
+den
+denn
+denselben
+depois
+der
+derer
+derselbe
+derselben
+des
+describe
+desde
+després
+desselben
+dessen
+detail
+di
+dich
+die
+dies
+diese
+dieselbe
+dieselben
+diesem
+diesen
+dieser
+dieses
+dins
+dintre
+dir
+do
+doch
+donat
+doncs
+donde
+done
+dort
+dos
+dov
+dove
+down
+du
+due
+d'un
+d'una
+d'unes
+d'uns
+durant
+durante
+durch
+during
+e
+each
+eûmes
+eût
+eûtes
+ebbe
+ebbero
+ebbi
+ed
+eg
+eh
+eight
+ein
+eine
+einem
+einen
+einer
+eines
+einig
+einige
+einigem
+einigen
+einiger
+einiges
+einmal
+either
+el
+ela
+elas
+ele
+eles
+eleven
+ella
+ellas
+elle
+ellos
+els
+else
+elsewhere
+em
+empty
+en
+encara
+enough
+ens
+entre
+er
+era
+erais
+eram
+eran
+erano
+eras
+eravamo
+eravate
+eren
+eres
+eri
+ero
+es
+esa
+esas
+ese
+eso
+esos
+essa
+essas
+esse
+essendo
+esses
+est
+esta
+está
+esté
+estaba
+estabais
+estábamos
+estaban
+estabas
+estad
+estada
+estadas
+estado
+estados
+estáis
+estéis
+estamos
+están
+estén
+estando
+estão
+estar
+estará
+estaré
+estaría
+estaríais
+estaríamos
+estarían
+estarías
+estaréis
+estarán
+estarás
+estaremos
+estas
+estás
+estés
+estava
+estavam
+estávamos
+estaven
+està
+estàvem
+estàveu
+este
+esteja
+estejam
+estejamos
+estemos
+estes
+esteu
+esteve
+estivéramos
+estivéssemos
+estive
+estivemos
+estiver
+estivera
+estiveram
+estiverem
+estivermos
+estivesse
+estivessem
+esto
+estos
+estou
+estoy
+estuve
+estuviéramos
+estuviésemos
+estuviera
+estuvierais
+estuvieran
+estuvieras
+estuvieron
+estuviese
+estuvieseis
+estuviesen
+estuvieses
+estuvimos
+estuviste
+estuvisteis
+estuvo
+et
+etc
+ets
+etwas
+eu
+euch
+eue
+euer
+eues
+eure
+eurem
+euren
+eurent
+eurer
+eures
+eus
+eusse
+eussent
+eusses
+eussiez
+eussions
+eut
+eux
+even
+ever
+every
+everyone
+everything
+everywhere
+except
+für
+faccia
+facciamo
+facciano
+facciate
+faccio
+facemmo
+facendo
+facesse
+facessero
+facessi
+facessimo
+faceste
+facesti
+faceva
+facevamo
+facevano
+facevate
+facevi
+facevo
+fai
+fûmes
+fanno
+farò
+farai
+fôramos
+faranno
+farà
+farebbe
+farebbero
+farei
+faremmo
+faremo
+fareste
+faresti
+farete
+fôssemos
+fût
+fûtes
+fece
+fecero
+feci
+few
+fifteen
+fify
+fill
+find
+fins
+fire
+first
+five
+foi
+fomos
+for
+fora
+foram
+forem
+former
+formerly
+formos
+forty
+fosse
+fossem
+fossero
+fossi
+fossimo
+foste
+fosti
+found
+four
+from
+front
+fu
+fuéramos
+fuésemos
+fue
+fuera
+fuerais
+fueran
+fueras
+fueron
+fuese
+fueseis
+fuesen
+fueses
+fui
+fuimos
+fuiste
+fuisteis
+full
+fummo
+furent
+furono
+further
+fus
+fusse
+fussent
+fusses
+fussiez
+fussions
+fut
+gairebé
+gegen
+get
+gewesen
+give
+gli
+go
+ha
+há
+hab
+había
+habíais
+habíamos
+habían
+habías
+habéis
+habe
+haben
+habida
+habidas
+habido
+habidos
+habiendo
+habrá
+habré
+habría
+habríais
+habríamos
+habrían
+habrías
+habréis
+habrán
+habrás
+habremos
+had
+hai
+haja
+hajam
+hajamos
+han
+hanno
+hão
+has
+hasnt
+hasta
+hat
+hatte
+hatten
+have
+havemos
+havia
+hay
+haya
+hayáis
+hayamos
+hayan
+hayas
+he
+hei
+hem
+hemos
+hence
+her
+here
+hereafter
+hereby
+herein
+hereupon
+hers
+herself
+heu
+hi 
+hier
+him
+himself
+hin
+hinter
+his
+ho
+houvéramos
+houvéssemos
+houve
+houvemos
+houver
+houvera
+houverá
+houveríamos
+houveram
+houverão
+houverei
+houverem
+houveremos
+houveria
+houveriam
+houvermos
+houvesse
+houvessem
+how
+however
+hube
+hubiéramos
+hubiésemos
+hubiera
+hubierais
+hubieran
+hubieras
+hubieron
+hubiese
+hubieseis
+hubiesen
+hubieses
+hubimos
+hubiste
+hubisteis
+hubo
+hundred
+i
+ich
+ici
+ie
+if
+igual
+iguals
+ihm
+ihn
+ihnen
+ihr
+ihre
+ihrem
+ihren
+ihrer
+ihres
+il
+ils
+im
+in
+inc
+indeed
+indem
+ins
+interest
+into
+io
+is
+isso
+ist
+isto
+it
+its
+itself
+j
+ja
+já
+je
+jede
+jedem
+jeden
+jeder
+jedes
+jene
+jenem
+jenen
+jener
+jenes
+jetzt
+kann
+können
+könnte
+keep
+kein
+keine
+keinem
+keinen
+keiner
+keines
+l
+la
+las
+last
+latter
+latterly
+le
+least
+lei
+les
+less
+leur
+leurs
+lhe
+lhes
+l'hi
+li
+li'n
+llavors
+lo
+loro
+los
+ltd
+lui
+m
+ma
+mí
+mía
+même
+mías
+machen
+made
+mais
+mal
+malgrat
+man
+manche
+manchem
+manchen
+mancher
+manches
+many
+mío
+míos
+mas
+más
+més
+mateix
+mateixa
+mateixes
+mateixos
+may
+me
+meanwhile
+mein
+meine
+meinem
+meinen
+meiner
+meines
+mentre
+mes
+mesmo
+meu
+meus
+meva
+meves
+m'he
+mi
+mia
+mich
+mie
+miei
+might
+mill
+mine
+minha
+minhas
+mio
+mir
+mis
+mit
+moi
+molt
+molta
+moltes
+molts
+mon
+mons
+more
+moreover
+most
+mostly
+move
+much
+mucho
+muchos
+muito
+muss
+musste
+must
+muy
+my
+myself
+n
+na
+nós
+nach
+nada
+name
+namely
+não
+nas
+ne
+negl
+negli
+nei
+neither
+nel
+nell
+nella
+nelle
+nello
+nem
+never
+nevertheless
+next
+n'he
+n'hi
+ni
+nicht
+nichts
+nine
+no
+nobody
+noch
+nogensmenys
+noi
+només
+non
+none
+noone
+nor
+nos
+nosaltres
+nosotras
+nosotros
+nossa
+nossas
+nosso
+nossos
+nostra
+nostre
+nostres
+nostri
+nostro
+not
+nothing
+notre
+nous
+now
+nowhere
+nuestra
+nuestras
+nuestro
+nuestros
+num
+numa
+nun
+nur
+o
+ob
+oder
+of
+off
+often
+oh
+ohne
+oi
+on
+once
+one
+only
+ont
+onto
+or
+os
+other
+others
+otherwise
+otra
+otras
+otro
+otros
+ou
+our
+ours
+ourselves
+out
+over
+own
+par
+para
+part
+pas
+pel
+pela
+pelas
+pelo
+pelos
+pels
+per
+però
+perché
+perhaps
+pero
+perquè
+più
+please
+poc 
+poca
+poco
+pocs
+poques
+por
+porque
+potser
+pour
+propi
+put
+qu
+què
+qué
+qual
+quale
+quals
+quan
+quando
+quant 
+quanta
+quante
+quanti
+quanto
+que
+quel
+quelcom
+quella
+quelle
+quelles
+quelli
+quello
+quels
+quem
+questa
+queste
+questi
+questo
+qui
+quien
+quienes
+quin
+quina
+quines
+quins
+rather
+re
+s
+sa
+sí
+só
+sóc
+són
+same
+sans
+são
+sarò
+sarai
+saranno
+sarà
+sarebbe
+sarebbero
+sarei
+saremmo
+saremo
+sareste
+saresti
+sarete
+se
+sea
+seáis
+seamos
+sean
+seas
+see
+seem
+seemed
+seeming
+seems
+sehr
+sei
+sein
+seine
+seinem
+seinen
+seiner
+seines
+seja
+sejam
+sejamos
+selbst
+sem
+semblant
+semblants
+sera
+será
+seré
+sería
+seríais
+seríamos
+serían
+serías
+serai
+seraient
+serais
+seréis
+serait
+serán
+serão
+seras
+serás
+serei
+seremos
+serez
+seria
+seriam
+seriez
+serions
+serious
+serons
+seront
+ses
+seu
+seu 
+seus
+seva
+several
+seves
+s'ha
+s'han
+she
+should
+show
+si
+sia
+siamo
+siano
+siate
+sich
+side
+sido
+sie
+siendo
+siete
+sin
+since
+sincere
+sind
+six
+sixty
+so
+sobre
+sobretot
+soi
+soient
+sois
+soit
+solament
+solche
+solchem
+solchen
+solcher
+solches
+soll
+sollte
+sols
+some
+somehow
+someone
+something
+sometime
+sometimes
+somewhere
+sommes
+somos
+son
+son 
+sondern
+sono
+sons 
+sonst
+sont
+sota
+sou
+soy
+soyez
+soyons
+sta
+stai
+stando
+stanno
+starò
+starai
+staranno
+starà
+starebbe
+starebbero
+starei
+staremmo
+staremo
+stareste
+staresti
+starete
+stava
+stavamo
+stavano
+stavate
+stavi
+stavo
+stemmo
+stesse
+stessero
+stessi
+stessimo
+steste
+stesti
+stette
+stettero
+stetti
+stia
+stiamo
+stiano
+stiate
+still
+sto
+su
+sua
+suas
+such
+sue
+sugl
+sugli
+sui
+suis
+sul
+sull
+sulla
+sulle
+sullo
+suo
+suoi
+sur
+sus
+suya
+suyas
+suyo
+suyos
+system
+t
+ta
+take
+tal
+tém
+també
+também
+también
+tampoc
+tan
+tínhamos
+tant
+tanta
+tantes
+tanto
+tú
+te
+tem
+temos
+ten
+tenía
+teníais
+teníamos
+tenían
+tenías
+tenéis
+tendrá
+tendré
+tendría
+tendríais
+tendríamos
+tendrían
+tendrías
+tendréis
+tendrán
+tendrás
+tendremos
+tened
+tenemos
+tenga
+tengáis
+tengamos
+tengan
+tengas
+tengo
+tenha
+tenham
+tenhamos
+tenho
+tenida
+tenidas
+tenido
+tenidos
+teniendo
+terá
+teríamos
+terão
+terei
+teremos
+teria
+teriam
+tes
+teu
+teus
+teva
+teve
+teves
+t'ha
+than
+t'han
+that
+the
+t'he
+their
+them
+themselves
+then
+thence
+there
+thereafter
+thereby
+therefore
+therein
+thereupon
+these
+they
+thick
+thin
+third
+this
+those
+though
+three
+through
+throughout
+thru
+thus
+ti
+tiene
+tienen
+tienes
+tinha
+tinham
+tivéramos
+tivéssemos
+tive
+tivemos
+tiver
+tivera
+tiveram
+tiverem
+tivermos
+tivesse
+tivessem
+to
+todo
+todos
+together
+toi
+ton
+tons
+too
+top
+tot
+tota
+totes
+tots
+toward
+towards
+tra
+tu
+tua
+tuas
+tue
+tuo
+tuoi
+tus
+tutti
+tutto
+tuve
+tuviéramos
+tuviésemos
+tuviera
+tuvierais
+tuvieran
+tuvieras
+tuvieron
+tuviese
+tuvieseis
+tuviesen
+tuvieses
+tuvimos
+tuviste
+tuvisteis
+tuvo
+tuya
+tuyas
+tuyo
+tuyos
+twelve
+twenty
+two
+um
+uma
+un
+una
+und
+under
+une
+unes
+uno
+unos
+uns
+unse
+unsem
+unsen
+unser
+unses
+unter
+until
+up
+upon
+us
+va
+vaig
+vam
+van
+vas
+very
+veu
+vi
+via
+viel
+você
+vocês
+voi
+vom
+von
+vor
+vos
+vosaltres
+vosotras
+vosotros
+vostra
+vostre
+vostres
+vostri
+vostro
+votre
+vous
+vuestra
+vuestras
+vuestro
+vuestros
+würde
+würden
+während
+war
+waren
+warst
+was
+we
+weg
+weil
+weiter
+welche
+welchem
+welchen
+welcher
+welches
+well
+wenn
+werde
+werden
+were
+what
+whatever
+when
+whence
+whenever
+where
+whereafter
+whereas
+whereby
+wherein
+whereupon
+wherever
+whether
+which
+while
+whither
+who
+whoever
+whole
+whom
+whose
+why
+wie
+wieder
+will
+wir
+wird
+wirst
+with
+within
+without
+wo
+wollen
+wollte
+would
+y
+ya
+yet
+yo
+you
+your
+yours
+yourself
+yourselvesa
+zu
+zum
+zur
+zwar
+zwischen
diff --git a/src/main/resources/com/gravity/goose/text/ca.txt b/src/main/resources/com/gravity/goose/text/ca.txt
new file mode 100644
index 000000000..458ecf212
--- /dev/null
+++ b/src/main/resources/com/gravity/goose/text/ca.txt
@@ -0,0 +1,219 @@
+a
+abans
+ací
+ah
+així
+això
+al
+als
+aleshores
+algun
+alguna
+algunes
+alguns
+alhora
+allà
+allí
+allò
+altra
+altre
+altres
+amb
+ambdós
+ambdues
+apa
+aquell
+aquella
+aquelles
+aquells
+aquest
+aquesta
+aquestes
+aquests
+aquí
+baix
+cada
+cadascú
+cadascuna
+cadascunes
+cadascuns
+com
+contra
+d'un
+d'una
+d'unes
+d'uns
+dalt
+de
+del
+dels
+des
+després
+dins
+dintre
+donat
+doncs
+durant
+e
+eh
+el
+els
+em
+en
+encara
+ens
+entre
+érem
+eren
+éreu
+es
+és
+esta
+està
+estàvem
+estaven
+estàveu
+esteu
+et
+etc
+ets
+fins
+fora
+gairebé
+ha
+han
+has
+havia
+he
+hem
+heu
+hi 
+ho
+i
+igual
+iguals
+ja
+l'hi
+la
+les
+li
+li'n
+llavors
+m'he
+ma
+mal
+malgrat
+mateix
+mateixa
+mateixes
+mateixos
+me
+mentre
+més
+meu
+meus
+meva
+meves
+molt
+molta
+moltes
+molts
+mon
+mons
+n'he
+n'hi
+ne
+ni
+no
+nogensmenys
+només
+nosaltres
+nostra
+nostre
+nostres
+o
+oh
+oi
+on
+pas
+pel
+pels
+per
+però
+perquè
+poc 
+poca
+pocs
+poques
+potser
+propi
+qual
+quals
+quan
+quant 
+que
+què
+quelcom
+qui
+quin
+quina
+quines
+quins
+s'ha
+s'han
+sa
+semblant
+semblants
+ses
+seu 
+seus
+seva
+seva
+seves
+si
+sobre
+sobretot
+sóc
+solament
+sols
+son 
+són
+sons 
+sota
+sou
+t'ha
+t'han
+t'he
+ta
+tal
+també
+tampoc
+tan
+tant
+tanta
+tantes
+teu
+teus
+teva
+teves
+ton
+tons
+tot
+tota
+totes
+tots
+un
+una
+unes
+uns
+us
+va
+vaig
+vam
+van
+vas
+veu
+vosaltres
+vostra
+vostre
+vostres
diff --git a/src/main/resources/com/gravity/goose/text/de.txt b/src/main/resources/com/gravity/goose/text/de.txt
new file mode 100644
index 000000000..721d5ca68
--- /dev/null
+++ b/src/main/resources/com/gravity/goose/text/de.txt
@@ -0,0 +1,231 @@
+über
+aber
+alle
+allem
+allen
+aller
+alles
+als
+also
+am
+an
+ander
+andere
+anderem
+anderen
+anderer
+anderes
+anderm
+andern
+anderr
+anders
+auch
+auf
+aus
+bei
+bin
+bis
+bist
+da
+daß
+damit
+dann
+das
+dasselbe
+dazu
+dein
+deine
+deinem
+deinen
+deiner
+deines
+dem
+demselben
+den
+denn
+denselben
+der
+derer
+derselbe
+derselben
+des
+desselben
+dessen
+dich
+die
+dies
+diese
+dieselbe
+dieselben
+diesem
+diesen
+dieser
+dieses
+dir
+doch
+dort
+du
+durch
+ein
+eine
+einem
+einen
+einer
+eines
+einig
+einige
+einigem
+einigen
+einiger
+einiges
+einmal
+er
+es
+etwas
+euch
+euer
+eure
+eurem
+euren
+eurer
+eures
+für
+gegen
+gewesen
+hab
+habe
+haben
+hat
+hatte
+hatten
+hier
+hin
+hinter
+ich
+ihm
+ihn
+ihnen
+ihr
+ihre
+ihrem
+ihren
+ihrer
+ihres
+im
+in
+indem
+ins
+ist
+jede
+jedem
+jeden
+jeder
+jedes
+jene
+jenem
+jenen
+jener
+jenes
+jetzt
+kann
+können
+könnte
+kein
+keine
+keinem
+keinen
+keiner
+keines
+machen
+man
+manche
+manchem
+manchen
+mancher
+manches
+mein
+meine
+meinem
+meinen
+meiner
+meines
+mich
+mir
+mit
+muss
+musste
+nach
+nicht
+nichts
+noch
+nun
+nur
+ob
+oder
+ohne
+sehr
+sein
+seine
+seinem
+seinen
+seiner
+seines
+selbst
+sich
+sie
+sind
+so
+solche
+solchem
+solchen
+solcher
+solches
+soll
+sollte
+sondern
+sonst
+um
+und
+uns
+unse
+unsem
+unsen
+unser
+unses
+unter
+viel
+vom
+von
+vor
+würde
+würden
+während
+war
+waren
+warst
+was
+weg
+weil
+weiter
+welche
+welchem
+welchen
+welcher
+welches
+wenn
+werde
+werden
+wie
+wieder
+will
+wir
+wird
+wirst
+wo
+wollen
+wollte
+zu
+zum
+zur
+zwar
+zwischen
diff --git a/src/main/resources/com/gravity/goose/text/en.txt b/src/main/resources/com/gravity/goose/text/en.txt
new file mode 100644
index 000000000..d3a39543f
--- /dev/null
+++ b/src/main/resources/com/gravity/goose/text/en.txt
@@ -0,0 +1,546 @@
+a's
+able
+about
+above
+according
+accordingly
+across
+actually
+after
+afterwards
+again
+against
+ain't
+all
+allow
+allows
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+and
+another
+any
+anybody
+anyhow
+anyone
+anything
+anyway
+anyways
+anywhere
+apart
+appear
+appreciate
+appropriate
+are
+aren't
+around
+as
+aside
+ask
+asking
+associated
+at
+available
+away
+awfully
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+behind
+being
+believe
+below
+beside
+besides
+best
+better
+between
+beyond
+both
+brief
+but
+by
+c
+c'mon
+c's
+came
+campaign
+can
+can't
+cannot
+cant
+cause
+causes
+certain
+certainly
+changes
+clearly
+co
+com
+come
+comes
+concerning
+consequently
+consider
+considering
+contain
+containing
+contains
+corresponding
+could
+couldn't
+course
+currently
+definitely
+described
+despite
+did
+didn't
+different
+do
+does
+doesn't
+doing
+don't
+done
+down
+downwards
+during
+each
+edu
+eight
+either
+else
+elsewhere
+enough
+endorsed
+entirely
+especially
+et
+etc
+even
+ever
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exactly
+example
+except
+far
+few
+fifth
+first
+financial
+five
+followed
+following
+follows
+for
+former
+formerly
+forth
+four
+from
+further
+furthermore
+get
+gets
+getting
+given
+gives
+go
+goes
+going
+gone
+got
+gotten
+greetings
+had
+hadn't
+happens
+hardly
+has
+hasn't
+have
+haven't
+having
+he
+he's
+hello
+help
+hence
+her
+here
+here's
+hereafter
+hereby
+herein
+hereupon
+hers
+herself
+hi
+him
+himself
+his
+hither
+hopefully
+how
+howbeit
+however
+i'd
+i'll
+i'm
+i've
+if
+ignored
+immediate
+in
+inasmuch
+inc
+indeed
+indicate
+indicated
+indicates
+inner
+insofar
+instead
+into
+inward
+is
+isn't
+it
+it'd
+it'll
+it's
+its
+itself
+just
+keep
+keeps
+kept
+know
+knows
+known
+last
+lately
+later
+latter
+latterly
+least
+less
+lest
+let
+let's
+like
+liked
+likely
+little
+look
+looking
+looks
+ltd
+mainly
+many
+may
+maybe
+me
+mean
+meanwhile
+merely
+might
+more
+moreover
+most
+mostly
+much
+must
+my
+myself
+name
+namely
+nd
+near
+nearly
+necessary
+need
+needs
+neither
+never
+nevertheless
+new
+next
+nine
+no
+nobody
+non
+none
+noone
+nor
+normally
+not
+nothing
+novel
+now
+nowhere
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+on
+once
+one
+ones
+only
+onto
+or
+other
+others
+otherwise
+ought
+our
+ours
+ourselves
+out
+outside
+over
+overall
+own
+particular
+particularly
+per
+perhaps
+placed
+please
+plus
+possible
+presumably
+probably
+provides
+quite
+quote
+quarterly
+rather
+really
+reasonably
+regarding
+regardless
+regards
+relatively
+respectively
+right
+said
+same
+saw
+say
+saying
+says
+second
+secondly
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sensible
+sent
+serious
+seriously
+seven
+several
+shall
+she
+should
+shouldn't
+since
+six
+so
+some
+somebody
+somehow
+someone
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+sorry
+specified
+specify
+specifying
+still
+sub
+such
+sup
+sure
+t's
+take
+taken
+tell
+tends
+than
+thank
+thanks
+thanx
+that
+that's
+thats
+the
+their
+theirs
+them
+themselves
+then
+thence
+there
+there's
+thereafter
+thereby
+therefore
+therein
+theres
+thereupon
+these
+they
+they'd
+they'll
+they're
+they've
+think
+third
+this
+thorough
+thoroughly
+those
+though
+three
+through
+throughout
+thru
+thus
+to
+together
+too
+took
+toward
+towards
+tried
+tries
+truly
+try
+trying
+twice
+two
+under
+unfortunately
+unless
+unlikely
+until
+unto
+up
+upon
+us
+use
+used
+useful
+uses
+using
+usually
+uucp
+value
+various
+very
+via
+viz
+vs
+want
+wants
+was
+wasn't
+way
+we
+we'd
+we'll
+we're
+we've
+welcome
+well
+went
+were
+weren't
+what
+what's
+whatever
+when
+whence
+whenever
+where
+where's
+whereafter
+whereas
+whereby
+wherein
+whereupon
+wherever
+whether
+which
+while
+whither
+who
+who's
+whoever
+whole
+whom
+whose
+why
+will
+willing
+wish
+with
+within
+without
+won't
+wonder
+would
+would
+wouldn't
+yes
+yet
+you
+you'd
+you'll
+you're
+you've
+your
+yours
+yourself
+yourselves
+zero
+official
+sharply
+criticized
\ No newline at end of file
diff --git a/src/main/resources/com/gravity/goose/text/es.txt b/src/main/resources/com/gravity/goose/text/es.txt
new file mode 100644
index 000000000..be3901a04
--- /dev/null
+++ b/src/main/resources/com/gravity/goose/text/es.txt
@@ -0,0 +1,308 @@
+a
+al
+él
+algo
+algunas
+algunos
+ante
+antes
+éramos
+como
+con
+contra
+cual
+cuando
+de
+del
+desde
+donde
+durante
+e
+el
+ella
+ellas
+ellos
+en
+entre
+era
+erais
+eran
+eras
+eres
+es
+esa
+esas
+ese
+eso
+esos
+esta
+está
+esté
+estaba
+estabais
+estábamos
+estaban
+estabas
+estad
+estada
+estadas
+estado
+estados
+estáis
+estéis
+estamos
+están
+estén
+estando
+estar
+estará
+estaré
+estaría
+estaríais
+estaríamos
+estarían
+estarías
+estaréis
+estarán
+estarás
+estaremos
+estas
+estás
+estés
+este
+estemos
+esto
+estos
+estoy
+estuve
+estuviéramos
+estuviésemos
+estuviera
+estuvierais
+estuvieran
+estuvieras
+estuvieron
+estuviese
+estuvieseis
+estuviesen
+estuvieses
+estuvimos
+estuviste
+estuvisteis
+estuvo
+fuéramos
+fuésemos
+fue
+fuera
+fuerais
+fueran
+fueras
+fueron
+fuese
+fueseis
+fuesen
+fueses
+fui
+fuimos
+fuiste
+fuisteis
+ha
+había
+habíais
+habíamos
+habían
+habías
+habéis
+habida
+habidas
+habido
+habidos
+habiendo
+habrá
+habré
+habría
+habríais
+habríamos
+habrían
+habrías
+habréis
+habrán
+habrás
+habremos
+han
+has
+hasta
+hay
+haya
+hayáis
+hayamos
+hayan
+hayas
+he
+hemos
+hube
+hubiéramos
+hubiésemos
+hubiera
+hubierais
+hubieran
+hubieras
+hubieron
+hubiese
+hubieseis
+hubiesen
+hubieses
+hubimos
+hubiste
+hubisteis
+hubo
+la
+las
+le
+les
+lo
+los
+mí
+mía
+mías
+mío
+míos
+más
+me
+mi
+mis
+mucho
+muchos
+muy
+nada
+ni
+no
+nos
+nosotras
+nosotros
+nuestra
+nuestras
+nuestro
+nuestros
+o
+os
+otra
+otras
+otro
+otros
+para
+pero
+poco
+por
+porque
+qué
+que
+quien
+quienes
+sí
+se
+sea
+seáis
+seamos
+sean
+seas
+será
+seré
+sería
+seríais
+seríamos
+serían
+serías
+seréis
+serán
+serás
+seremos
+sido
+siendo
+sin
+sobre
+sois
+somos
+son
+soy
+su
+sus
+suya
+suyas
+suyo
+suyos
+también
+tanto
+tú
+te
+tenía
+teníais
+teníamos
+tenían
+tenías
+tenéis
+tendrá
+tendré
+tendría
+tendríais
+tendríamos
+tendrían
+tendrías
+tendréis
+tendrán
+tendrás
+tendremos
+tened
+tenemos
+tenga
+tengáis
+tengamos
+tengan
+tengas
+tengo
+tenida
+tenidas
+tenido
+tenidos
+teniendo
+ti
+tiene
+tienen
+tienes
+todo
+todos
+tu
+tus
+tuve
+tuviéramos
+tuviésemos
+tuviera
+tuvierais
+tuvieran
+tuvieras
+tuvieron
+tuviese
+tuvieseis
+tuviesen
+tuvieses
+tuvimos
+tuviste
+tuvisteis
+tuvo
+tuya
+tuyas
+tuyo
+tuyos
+un
+una
+uno
+unos
+vosotras
+vosotros
+vuestra
+vuestras
+vuestro
+vuestros
+y
+ya
+yo
diff --git a/src/main/resources/com/gravity/goose/text/fr.txt b/src/main/resources/com/gravity/goose/text/fr.txt
new file mode 100644
index 000000000..9a0bda2a2
--- /dev/null
+++ b/src/main/resources/com/gravity/goose/text/fr.txt
@@ -0,0 +1,163 @@
+êtes
+ai
+aie
+aient
+aies
+ait
+as
+été
+étée
+étées
+étaient
+étais
+était
+étant
+étés
+étiez
+étions
+au
+aura
+aurai
+auraient
+aurais
+aurait
+auras
+aurez
+auriez
+aurions
+aurons
+auront
+aux
+avaient
+avais
+avait
+avec
+avez
+aviez
+avions
+avons
+ayant
+ayez
+ayons
+à
+c
+ce
+ceci
+celà
+ces
+cet
+cette
+d
+dans
+de
+des
+du
+eûmes
+eût
+eûtes
+elle
+en
+es
+est
+et
+eu
+eue
+eues
+eurent
+eus
+eusse
+eussent
+eusses
+eussiez
+eussions
+eut
+eux
+fûmes
+fût
+fûtes
+furent
+fus
+fusse
+fussent
+fusses
+fussiez
+fussions
+fut
+ici
+il
+ils
+j
+je
+l
+la
+le
+les
+leur
+leurs
+lui
+m
+ma
+même
+mais
+me
+mes
+moi
+mon
+n
+ne
+nos
+notre
+nous
+on
+ont
+ou
+par
+pas
+pour
+qu
+que
+quel
+quelle
+quelles
+quels
+qui
+s
+sa
+sans
+se
+sera
+serai
+seraient
+serais
+serait
+seras
+serez
+seriez
+serions
+serons
+seront
+ses
+soi
+soient
+sois
+soit
+sommes
+son
+sont
+soyez
+soyons
+suis
+sur
+t
+ta
+te
+tes
+toi
+ton
+tu
+un
+une
+vos
+votre
+vous
+y
diff --git a/src/main/resources/com/gravity/goose/text/it.txt b/src/main/resources/com/gravity/goose/text/it.txt
new file mode 100644
index 000000000..52fdd78a1
--- /dev/null
+++ b/src/main/resources/com/gravity/goose/text/it.txt
@@ -0,0 +1,279 @@
+a
+è
+abbia
+abbiamo
+abbiano
+abbiate
+ad
+agl
+agli
+ai
+al
+all
+alla
+alle
+allo
+anche
+avemmo
+avendo
+avesse
+avessero
+avessi
+avessimo
+aveste
+avesti
+avete
+aveva
+avevamo
+avevano
+avevate
+avevi
+avevo
+avrò
+avrai
+avranno
+avrà
+avrebbe
+avrebbero
+avrei
+avremmo
+avremo
+avreste
+avresti
+avrete
+avuta
+avute
+avuti
+avuto
+c
+che
+chi
+ci
+coi
+col
+come
+con
+contro
+cui
+da
+dagl
+dagli
+dai
+dal
+dall
+dalla
+dalle
+dallo
+degl
+degli
+dei
+del
+dell
+della
+delle
+dello
+di
+dov
+dove
+e
+ebbe
+ebbero
+ebbi
+ed
+era
+erano
+eravamo
+eravate
+eri
+ero
+essendo
+faccia
+facciamo
+facciano
+facciate
+faccio
+facemmo
+facendo
+facesse
+facessero
+facessi
+facessimo
+faceste
+facesti
+faceva
+facevamo
+facevano
+facevate
+facevi
+facevo
+fai
+fanno
+farò
+farai
+faranno
+farà
+farebbe
+farebbero
+farei
+faremmo
+faremo
+fareste
+faresti
+farete
+fece
+fecero
+feci
+fosse
+fossero
+fossi
+fossimo
+foste
+fosti
+fu
+fui
+fummo
+furono
+gli
+ha
+hai
+hanno
+ho
+i
+il
+in
+io
+l
+la
+le
+lei
+li
+lo
+loro
+lui
+ma
+mi
+mia
+mie
+miei
+mio
+ne
+negl
+negli
+nei
+nel
+nell
+nella
+nelle
+nello
+noi
+non
+nostra
+nostre
+nostri
+nostro
+o
+per
+perché
+più
+quale
+quanta
+quante
+quanti
+quanto
+quella
+quelle
+quelli
+quello
+questa
+queste
+questi
+questo
+sarò
+sarai
+saranno
+sarà
+sarebbe
+sarebbero
+sarei
+saremmo
+saremo
+sareste
+saresti
+sarete
+se
+sei
+si
+sia
+siamo
+siano
+siate
+siete
+sono
+sta
+stai
+stando
+stanno
+starò
+starai
+staranno
+starà
+starebbe
+starebbero
+starei
+staremmo
+staremo
+stareste
+staresti
+starete
+stava
+stavamo
+stavano
+stavate
+stavi
+stavo
+stemmo
+stesse
+stessero
+stessi
+stessimo
+steste
+stesti
+stette
+stettero
+stetti
+stia
+stiamo
+stiano
+stiate
+sto
+su
+sua
+sue
+sugl
+sugli
+sui
+sul
+sull
+sulla
+sulle
+sullo
+suo
+suoi
+ti
+tra
+tu
+tua
+tue
+tuo
+tuoi
+tutti
+tutto
+un
+una
+uno
+vi
+voi
+vostra
+vostre
+vostri
+vostro
diff --git a/src/main/resources/com/gravity/goose/text/pt.txt b/src/main/resources/com/gravity/goose/text/pt.txt
new file mode 100644
index 000000000..9723b3798
--- /dev/null
+++ b/src/main/resources/com/gravity/goose/text/pt.txt
@@ -0,0 +1,203 @@
+a
+ao
+aos
+aquela
+aquelas
+aquele
+aqueles
+aquilo
+éramos
+as
+até
+à
+às
+com
+como
+da
+das
+de
+dela
+delas
+dele
+deles
+depois
+do
+dos
+e
+ela
+elas
+ele
+eles
+em
+entre
+era
+eram
+essa
+essas
+esse
+esses
+esta
+está
+estamos
+estão
+estas
+estava
+estavam
+estávamos
+este
+esteja
+estejam
+estejamos
+estes
+esteve
+estivéramos
+estivéssemos
+estive
+estivemos
+estiver
+estivera
+estiveram
+estiverem
+estivermos
+estivesse
+estivessem
+estou
+eu
+fôramos
+fôssemos
+foi
+fomos
+for
+fora
+foram
+forem
+formos
+fosse
+fossem
+fui
+há
+haja
+hajam
+hajamos
+hão
+havemos
+hei
+houvéramos
+houvéssemos
+houve
+houvemos
+houver
+houvera
+houverá
+houveríamos
+houveram
+houverão
+houverei
+houverem
+houveremos
+houveria
+houveriam
+houvermos
+houvesse
+houvessem
+isso
+isto
+já
+lhe
+lhes
+mais
+mas
+me
+mesmo
+meu
+meus
+minha
+minhas
+muito
+na
+nós
+não
+nas
+nem
+no
+nos
+nossa
+nossas
+nosso
+nossos
+num
+numa
+o
+os
+ou
+para
+pela
+pelas
+pelo
+pelos
+por
+qual
+quando
+que
+quem
+só
+são
+se
+seja
+sejam
+sejamos
+sem
+será
+seríamos
+serão
+serei
+seremos
+seria
+seriam
+seu
+seus
+somos
+sou
+sua
+suas
+tém
+também
+tínhamos
+te
+tem
+temos
+tenha
+tenham
+tenhamos
+tenho
+terá
+teríamos
+terão
+terei
+teremos
+teria
+teriam
+teu
+teus
+teve
+tinha
+tinham
+tivéramos
+tivéssemos
+tive
+tivemos
+tiver
+tivera
+tiveram
+tiverem
+tivermos
+tivesse
+tivessem
+tu
+tua
+tuas
+um
+uma
+você
+vocês
+vos

From c6ccc8adde01b8832324aab2e54d0e605cb65488 Mon Sep 17 00:00:00 2001
From: Jesus Lanchas <jlanchas@acceso.com>
Date: Wed, 29 Feb 2012 14:22:45 +0100
Subject: [PATCH 2/2] Complement to the previous commit.

---
 pom.xml                                       |  11 +-
 .../com/gravity/goose/text/stopwords-en.txt   | 546 ------------------
 .../gravity/goose/network/HtmlFetcher.scala   |  56 +-
 .../com/gravity/goose/text/StopWords.scala    |  28 +-
 4 files changed, 56 insertions(+), 585 deletions(-)
 delete mode 100644 src/main/resources/com/gravity/goose/text/stopwords-en.txt

diff --git a/pom.xml b/pom.xml
index 9be4d60bf..b0607836d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -105,6 +105,15 @@
 
       </exclusions>
     </dependency>
+   
+
+    <dependency>
+	<groupId>com.googlecode.juniversalchardet</groupId>
+	<artifactId>juniversalchardet</artifactId>
+	<version>1.0.3</version>
+	<scope>compile</scope>
+    </dependency>
+
   </dependencies>
 
   <!--
@@ -190,4 +199,4 @@
     </plugins>
   </build>
 
-</project>
\ No newline at end of file
+</project>
diff --git a/src/main/resources/com/gravity/goose/text/stopwords-en.txt b/src/main/resources/com/gravity/goose/text/stopwords-en.txt
deleted file mode 100644
index d3a39543f..000000000
--- a/src/main/resources/com/gravity/goose/text/stopwords-en.txt
+++ /dev/null
@@ -1,546 +0,0 @@
-a's
-able
-about
-above
-according
-accordingly
-across
-actually
-after
-afterwards
-again
-against
-ain't
-all
-allow
-allows
-almost
-alone
-along
-already
-also
-although
-always
-am
-among
-amongst
-an
-and
-another
-any
-anybody
-anyhow
-anyone
-anything
-anyway
-anyways
-anywhere
-apart
-appear
-appreciate
-appropriate
-are
-aren't
-around
-as
-aside
-ask
-asking
-associated
-at
-available
-away
-awfully
-be
-became
-because
-become
-becomes
-becoming
-been
-before
-beforehand
-behind
-being
-believe
-below
-beside
-besides
-best
-better
-between
-beyond
-both
-brief
-but
-by
-c
-c'mon
-c's
-came
-campaign
-can
-can't
-cannot
-cant
-cause
-causes
-certain
-certainly
-changes
-clearly
-co
-com
-come
-comes
-concerning
-consequently
-consider
-considering
-contain
-containing
-contains
-corresponding
-could
-couldn't
-course
-currently
-definitely
-described
-despite
-did
-didn't
-different
-do
-does
-doesn't
-doing
-don't
-done
-down
-downwards
-during
-each
-edu
-eight
-either
-else
-elsewhere
-enough
-endorsed
-entirely
-especially
-et
-etc
-even
-ever
-every
-everybody
-everyone
-everything
-everywhere
-ex
-exactly
-example
-except
-far
-few
-fifth
-first
-financial
-five
-followed
-following
-follows
-for
-former
-formerly
-forth
-four
-from
-further
-furthermore
-get
-gets
-getting
-given
-gives
-go
-goes
-going
-gone
-got
-gotten
-greetings
-had
-hadn't
-happens
-hardly
-has
-hasn't
-have
-haven't
-having
-he
-he's
-hello
-help
-hence
-her
-here
-here's
-hereafter
-hereby
-herein
-hereupon
-hers
-herself
-hi
-him
-himself
-his
-hither
-hopefully
-how
-howbeit
-however
-i'd
-i'll
-i'm
-i've
-if
-ignored
-immediate
-in
-inasmuch
-inc
-indeed
-indicate
-indicated
-indicates
-inner
-insofar
-instead
-into
-inward
-is
-isn't
-it
-it'd
-it'll
-it's
-its
-itself
-just
-keep
-keeps
-kept
-know
-knows
-known
-last
-lately
-later
-latter
-latterly
-least
-less
-lest
-let
-let's
-like
-liked
-likely
-little
-look
-looking
-looks
-ltd
-mainly
-many
-may
-maybe
-me
-mean
-meanwhile
-merely
-might
-more
-moreover
-most
-mostly
-much
-must
-my
-myself
-name
-namely
-nd
-near
-nearly
-necessary
-need
-needs
-neither
-never
-nevertheless
-new
-next
-nine
-no
-nobody
-non
-none
-noone
-nor
-normally
-not
-nothing
-novel
-now
-nowhere
-obviously
-of
-off
-often
-oh
-ok
-okay
-old
-on
-once
-one
-ones
-only
-onto
-or
-other
-others
-otherwise
-ought
-our
-ours
-ourselves
-out
-outside
-over
-overall
-own
-particular
-particularly
-per
-perhaps
-placed
-please
-plus
-possible
-presumably
-probably
-provides
-quite
-quote
-quarterly
-rather
-really
-reasonably
-regarding
-regardless
-regards
-relatively
-respectively
-right
-said
-same
-saw
-say
-saying
-says
-second
-secondly
-see
-seeing
-seem
-seemed
-seeming
-seems
-seen
-self
-selves
-sensible
-sent
-serious
-seriously
-seven
-several
-shall
-she
-should
-shouldn't
-since
-six
-so
-some
-somebody
-somehow
-someone
-something
-sometime
-sometimes
-somewhat
-somewhere
-soon
-sorry
-specified
-specify
-specifying
-still
-sub
-such
-sup
-sure
-t's
-take
-taken
-tell
-tends
-than
-thank
-thanks
-thanx
-that
-that's
-thats
-the
-their
-theirs
-them
-themselves
-then
-thence
-there
-there's
-thereafter
-thereby
-therefore
-therein
-theres
-thereupon
-these
-they
-they'd
-they'll
-they're
-they've
-think
-third
-this
-thorough
-thoroughly
-those
-though
-three
-through
-throughout
-thru
-thus
-to
-together
-too
-took
-toward
-towards
-tried
-tries
-truly
-try
-trying
-twice
-two
-under
-unfortunately
-unless
-unlikely
-until
-unto
-up
-upon
-us
-use
-used
-useful
-uses
-using
-usually
-uucp
-value
-various
-very
-via
-viz
-vs
-want
-wants
-was
-wasn't
-way
-we
-we'd
-we'll
-we're
-we've
-welcome
-well
-went
-were
-weren't
-what
-what's
-whatever
-when
-whence
-whenever
-where
-where's
-whereafter
-whereas
-whereby
-wherein
-whereupon
-wherever
-whether
-which
-while
-whither
-who
-who's
-whoever
-whole
-whom
-whose
-why
-will
-willing
-wish
-with
-within
-without
-won't
-wonder
-would
-would
-wouldn't
-yes
-yet
-you
-you'd
-you'll
-you're
-you've
-your
-yours
-yourself
-yourselves
-zero
-official
-sharply
-criticized
\ No newline at end of file
diff --git a/src/main/scala/com/gravity/goose/network/HtmlFetcher.scala b/src/main/scala/com/gravity/goose/network/HtmlFetcher.scala
index 0691706ea..038e2508e 100644
--- a/src/main/scala/com/gravity/goose/network/HtmlFetcher.scala
+++ b/src/main/scala/com/gravity/goose/network/HtmlFetcher.scala
@@ -53,6 +53,9 @@ import java.util.Date
 import java.util.List
 import com.gravity.goose.utils.Logging
 import com.gravity.goose.Configuration
+import org.apache.commons.io.IOUtils
+
+import org.mozilla.universalchardet.UniversalDetector
 
 /**
  * User: Jim Plush
@@ -132,7 +135,7 @@ object HtmlFetcher extends Logging {
           }
         }
         try {
-          htmlResult = HtmlFetcher.convertStreamToString(instream, 15728640, encodingType).trim
+          htmlResult = HtmlFetcher.convertStreamToString(instream, encodingType).trim
         }
         finally {
           entity.consumeContent()
@@ -283,52 +286,39 @@ object HtmlFetcher extends Logging {
    * @param maxBytes The max bytes that we want to read from the input stream
    * @return String
    */
-  def convertStreamToString(is: InputStream, maxBytes: Int, encodingType: String): String = {
-    val buf: Array[Char] = new Array[Char](2048)
-    var r: Reader = null
-    val s = new StringBuilder
+  def convertStreamToString(is: InputStream, httpEncodingType: String): String = {
     try {
-      r = new InputStreamReader(is, encodingType)
-      var bytesRead: Int = 2048
-      var inLoop = true
-      while (inLoop) {
-        if (bytesRead >= maxBytes) {
-          throw new MaxBytesException
-        }
-        var n: Int = r.read(buf)
-        bytesRead += 2048
-
-        if (n < 0) inLoop = false
-        if (inLoop) s.appendAll(buf, 0, n)
-      }
-      return s.toString()
+      var buf : Array[Byte] = IOUtils.toByteArray(is)
+      var finalEncodingType = detectEncoding(buf, httpEncodingType)
+      return new String(buf, finalEncodingType)
     }
     catch {
       case e: SocketTimeoutException => {
         logger.warn(e.toString + " " + e.getMessage)
       }
       case e: UnsupportedEncodingException => {
-        logger.warn(e.toString + " Encoding: " + encodingType)
+        logger.warn(e.toString + " " + e.getMessage)
       }
       case e: IOException => {
         logger.warn(e.toString + " " + e.getMessage)
       }
     }
-    finally {
-      if (r != null) {
-        try {
-          r.close()
-        }
-        catch {
-          case e: Exception => {
-          }
-        }
-      }
-    }
     null
   }
-
-
+  
+  def detectEncoding(buf : Array[Byte], httpEncodingType : String) : String = {
+    var detector : UniversalDetector = new UniversalDetector(null)
+    detector.handleData(buf, 0, buf.size)
+    detector.dataEnd()
+    
+    var detectedEncoding = detector.getDetectedCharset()
+    
+    if(detectedEncoding != null) {
+    	return detectedEncoding
+    } else {
+    	return httpEncodingType
+    }
+  }
 }
 
 
diff --git a/src/main/scala/com/gravity/goose/text/StopWords.scala b/src/main/scala/com/gravity/goose/text/StopWords.scala
index 9f8669e8b..0f1332614 100644
--- a/src/main/scala/com/gravity/goose/text/StopWords.scala
+++ b/src/main/scala/com/gravity/goose/text/StopWords.scala
@@ -27,13 +27,20 @@ package com.gravity.goose.text
 import java.util._
 import com.gravity.goose.utils.FileHelper
 
-object StopWords {
+import scala.collection.immutable.Map
 
+object StopWords {
+  
   // the confusing pattern below is basically just match any non-word character excluding white-space.
   private val PUNCTUATION: StringReplacement = StringReplacement.compile("[^\\p{Ll}\\p{Lu}\\p{Lt}\\p{Lo}\\p{Nd}\\p{Pc}\\s]", string.empty)
-
-  val STOP_WORDS = FileHelper.loadResourceFile("stopwords-en.txt", StopWords.getClass).split("\n").toSet
-
+  
+  val STOP_WORDS = Map("ca" -> FileHelper.loadResourceFile("ca.txt", StopWords.getClass).split("\n").toSet,
+		  				"de" -> FileHelper.loadResourceFile("de.txt", StopWords.getClass).split("\n").toSet,
+		  				"en" -> FileHelper.loadResourceFile("en.txt", StopWords.getClass).split("\n").toSet,
+		  				"es" -> FileHelper.loadResourceFile("es.txt", StopWords.getClass).split("\n").toSet,
+		  				"fr" -> FileHelper.loadResourceFile("fr.txt", StopWords.getClass).split("\n").toSet,
+		  				"it" -> FileHelper.loadResourceFile("it.txt", StopWords.getClass).split("\n").toSet,
+		  				"pt" -> FileHelper.loadResourceFile("pt.txt", StopWords.getClass).split("\n").toSet);
 
   def removePunctuation(str: String): String = {
     PUNCTUATION.replaceAll(str)
@@ -48,15 +55,26 @@ object StopWords {
     val candidateWords: Array[String] = string.SPACE_SPLITTER.split(strippedInput)
 
     val overlappingStopWords: List[String] = new ArrayList[String]
+    
+    val languageCode : String = detectLanguage(content)
+    val stopWords = STOP_WORDS(languageCode)
 
     candidateWords.foreach(w => {
-       if (STOP_WORDS.contains(w.toLowerCase)) overlappingStopWords.add(w.toLowerCase)
+       if (stopWords.contains(w.toLowerCase)) overlappingStopWords.add(w.toLowerCase)
     })
     ws.setWordCount(candidateWords.length)
     ws.setStopWordCount(overlappingStopWords.size)
     ws.setStopWords(overlappingStopWords)
     ws
   }
+  
+  /**
+   * This method returns the code of the language identified in the content
+   * passed as parameter.
+   */
+  def detectLanguage(content: String): String = {
+    return "en" // TODO Fixme using some automatic language detector library
+  }
 
 
 }