diff --git a/.travis.yml b/.travis.yml index c075f646..6166d6e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ script: > cd target && git clone https://github.com/lemonlabsuk/scala-uri-demo.git && cd scala-uri-demo && - sbt -Dscala.ver=$TRAVIS_SCALA_VERSION -Dscala.uri.ver=3.0.0 test && + sbt -Dscala.ver=$TRAVIS_SCALA_VERSION -Dscala.uri.ver=3.1.0 test && cd "$TRAVIS_BUILD_DIR" jdk: diff --git a/README.md b/README.md index f8eeed32..f495e3fc 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ To include it in your SBT project from maven central: ```scala -"io.lemonlabs" %% "scala-uri" % "3.0.0" +"io.lemonlabs" %% "scala-uri" % "3.1.0" ``` ## Migration Guides @@ -190,6 +190,34 @@ val relativeUrl = Url.parse("/example?a=b") relativeUrl.withScheme("http").withHost("www.example.com") // This is http://www.example.com/example?a=b ``` +## Redacting URLs + +It is possible to print out redacted URLs to logs with sensitive information either removed or replaced with a placeholder + +Replacing with a placeholder: + +```scala mdoc:reset +import io.lemonlabs.uri._ +import io.lemonlabs.uri.redact._ + +val url = Url.parse("http://user:password@example.com?secret=123&last=yes") + +// This returns http://xxx:xxx@example.com?secret=xxx&last=yes +url.toRedactedString(Redact.withPlaceholder("xxx").params("secret", "other").user().password()) +``` + +Removing: + +```scala mdoc:reset +import io.lemonlabs.uri._ +import io.lemonlabs.uri.redact._ + +val url = Url.parse("http://user:password@example.com?secret=123&other=true") + +// This returns http://example.com +url.toRedactedString(Redact.byRemoving.allParams().userInfo()) +``` + ## Pattern Matching URIs ```scala mdoc:reset @@ -816,13 +844,13 @@ The type class instances exist in the companion objects for these types. * For `2.11.x` support use `scala-uri` `1.4.10` from branch [`1.4.x`](https://github.com/lemonlabsuk/scala-uri/tree/1.4.x) * For `2.10.x` support use `scala-uri` `0.4.17` from branch [`0.4.x`](https://github.com/lemonlabsuk/scala-uri/tree/0.4.x) * For `2.9.x` support use `scala-uri` `0.3.6` from branch [`0.3.x`](https://github.com/lemonlabsuk/scala-uri/tree/0.3.x) - * For Scala.js `1.x.x` support, use `scala-uri` `3.0.0` + * For Scala.js `1.x.x` support, use `scala-uri` `3.1.0` * For Scala.js `0.6.x` support, use `scala-uri` `2.2.3` Release builds are available in maven central. For SBT users just add the following dependency: ```scala -"io.lemonlabs" %% "scala-uri" % "3.0.0" +"io.lemonlabs" %% "scala-uri" % "3.1.0" ``` For maven users you should use (for 2.13.x): @@ -831,7 +859,7 @@ For maven users you should use (for 2.13.x): io.lemonlabs scala-uri_2.13 - 3.0.0 + 3.1.0 ``` diff --git a/build.sbt b/build.sbt index 3f95e80e..cdcecac5 100644 --- a/build.sbt +++ b/build.sbt @@ -22,7 +22,7 @@ val sharedSettings = Seq( organization := "io.lemonlabs", libraryDependencies ++= Seq( "org.typelevel" %%% "simulacrum-scalafix-annotations" % "0.5.4", - "org.scalatest" %%% "scalatest" % "3.2.5" % Test, + "org.scalatest" %%% "scalatest" % "3.2.6" % Test, "org.scalatestplus" %%% "scalacheck-1-14" % "3.2.2.0" % Test, "org.scalacheck" %%% "scalacheck" % "1.15.3" % Test, "org.typelevel" %%% "cats-laws" % "2.4.2" % Test @@ -116,7 +116,8 @@ val publishingSettings = Seq( val previousVersions = (0 to 0).map(v => s"3.$v.0").toSet val mimaExcludes = Seq( - ProblemFilters.exclude[ReversedMissingMethodProblem]("io.lemonlabs.uri.typesafe.QueryValueInstances1.*") + ProblemFilters.exclude[ReversedMissingMethodProblem]("io.lemonlabs.uri.typesafe.QueryValueInstances1.*"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("io.lemonlabs.uri.Url.*") ) val mimaSettings = Seq( diff --git a/bumpversion.sh b/bumpversion.sh index 46b87898..b339cfae 100755 --- a/bumpversion.sh +++ b/bumpversion.sh @@ -5,7 +5,7 @@ git add shared/src/main/scala/io/lemonlabs/uri/inet/PublicSuffixes.scala git commit -m"Update public suffixes" echo "Running SBT to determine current version. Please wait..." -VER=$(sbt 'project scalaUriJVM' 'show version' | tail -n2 | head -n1 | cut -f2 -d' ') +VER=$(sbt 'project scalaUriJVM' 'show version' | tail -n1 | cut -f2 -d' ') echo "Current version is $VER, what is the next version?" read -r NEW_VER diff --git a/project/plugins.sbt b/project/plugins.sbt index dfe7c1a9..99cfa218 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -12,4 +12,4 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2") addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.2.18") -addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.25") +addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.26") diff --git a/shared/src/main/scala/io/lemonlabs/uri/Uri.scala b/shared/src/main/scala/io/lemonlabs/uri/Uri.scala index 4fe6a27d..50ed2d69 100644 --- a/shared/src/main/scala/io/lemonlabs/uri/Uri.scala +++ b/shared/src/main/scala/io/lemonlabs/uri/Uri.scala @@ -4,6 +4,7 @@ import java.util.Base64 import cats.{Eq, Order, Show} import io.lemonlabs.uri.config.UriConfig import io.lemonlabs.uri.parsing.{UriParser, UrlParser, UrnParser} +import io.lemonlabs.uri.redact.Redactor import io.lemonlabs.uri.typesafe.{ Fragment, PathPart, @@ -175,7 +176,7 @@ sealed trait Url extends Uri { /** Copies this Url but with the authority set as the given value. * - * @param authority the authority host to set + * @param authority the authority to set * @return a new Url with the specified authority */ def withAuthority(authority: Authority): SelfWithAuthority @@ -333,6 +334,16 @@ sealed trait Url extends Uri { def removeQueryString(): Self = withQueryString(QueryString.empty) + /** Removes the user-info (both user and password) from this URL + * @return This URL without the user-info + */ + def removeUserInfo(): Self + + /** Removes any password from this URL's user-info + * @return This URL without the password + */ + def removePassword(): Self + /** Transforms the Query String by applying the specified PartialFunction to each Query String Parameter * * Parameters not defined in the PartialFunction will be left as-is. @@ -393,6 +404,16 @@ sealed trait Url extends Uri { def filterQueryNames(f: String => Boolean): Self = withQueryString(query.filterNames(f)) + /** Transforms this URL by applying the specified Function to the user if it exists + * @return + */ + def mapUser(f: String => String): Self + + /** Transforms this URL by applying the specified Function to the password if it exists + * @return + */ + def mapPassword(f: String => String): Self + /** Returns the apex domain for this URL. * * The apex domain is constructed from the public suffix for this URL's host prepended with the @@ -452,6 +473,9 @@ sealed trait Url extends Uri { case "" => "" case s => "?" + s } + + def toRedactedString(redactor: Redactor)(implicit conf: UriConfig = UriConfig.default): String = + redactor.apply(this).toString(conf) } object Url { @@ -553,6 +577,11 @@ final case class RelativeUrl(path: UrlPath, query: QueryString, fragment: Option private[uri] def toString(c: UriConfig): String = path.toString(c) + queryToString(c) + fragmentToString(c) + + def removeUserInfo(): RelativeUrl = this + def removePassword(): RelativeUrl = this + def mapUser(f: String => String): RelativeUrl = this + def mapPassword(f: String => String): RelativeUrl = this } object RelativeUrl { @@ -624,6 +653,41 @@ sealed trait UrlWithAuthority extends Url { def withPort(port: Int): Self = withAuthority(authority.copy(port = Some(port))) + def withUserInfo(ui: Option[UserInfo]): Self = + withAuthority(authority.copy(userInfo = ui)) + + /** Removes any user from this URL + * + * @return This URL without the user + */ + def removeUserInfo(): Self = + withAuthority(authority.copy(userInfo = None)) + + /** Removes any password from this URL + * + * @return This URL without the password + */ + def removePassword(): Self = + withUserInfo(userInfo.map(_.copy(password = None))) + + /** Transforms this URL by applying the specified Function to the user if it exists + * + * @return + */ + override def mapUser(f: String => String): Self = + user.fold(self) { u => + withUserInfo(userInfo.map(_.copy(user = f(u)))) + } + + /** Transforms this URL by applying the specified Function to the password if it exists + * + * @return + */ + override def mapPassword(f: String => String): Self = + password.fold(self) { p => + withUserInfo(userInfo.map(_.copy(password = Some(f(p))))) + } + /** Returns the longest public suffix for the host in this URI. Examples include: * `com` for `www.example.com` * `co.uk` for `www.example.co.uk` @@ -846,6 +910,11 @@ sealed trait UrlWithoutAuthority extends Url { def subdomains: Vector[String] = Vector.empty def shortestSubdomain: Option[String] = None def longestSubdomain: Option[String] = None + + def removeUserInfo(): Self = self + def removePassword(): Self = self + def mapUser(f: String => String): Self = self + def mapPassword(f: String => String): Self = self } object UrlWithoutAuthority { diff --git a/shared/src/main/scala/io/lemonlabs/uri/inet/PublicSuffixes.scala b/shared/src/main/scala/io/lemonlabs/uri/inet/PublicSuffixes.scala index 254ce3b9..a65a2c72 100644 --- a/shared/src/main/scala/io/lemonlabs/uri/inet/PublicSuffixes.scala +++ b/shared/src/main/scala/io/lemonlabs/uri/inet/PublicSuffixes.scala @@ -9,9 +9,7 @@ object PublicSuffixes { "city.nagoya.jp", "city.sapporo.jp", "city.sendai.jp", - "city.yokohama.jp", - "teams.algorithmia.com", - "test.algorithmia.com" + "city.yokohama.jp" ) lazy val wildcardPrefixes = Set( @@ -33,11 +31,9 @@ object PublicSuffixes { "np", "pg", "sch.uk", - "ye", "dev.adobeaemcloud.com", "compute.estate", "alces.network", - "algorithmia.com", "compute.amazonaws.com", "compute-1.amazonaws.com", "compute.amazonaws.com.cn", @@ -48,7 +44,9 @@ object PublicSuffixes { "banzai.cloud", "backyards.banzaicloud.io", "lcl.dev", + "lclstage.dev", "stg.dev", + "stgstage.dev", "otap.co", "cryptonomic.net", "customer-oci.com", @@ -78,6 +76,10 @@ object PublicSuffixes { "magentosite.cloud", "cloud.metacentrum.cz", "azurecontainer.io", + "northflank.app", + "code.run", + "webpaas.ovh.net", + "hosting.ovh.net", "owo.codes", "platformsh.site", "dweb.link", @@ -4210,13 +4212,14 @@ object PublicSuffixes { "edu.mx", "net.mx", "my", + "biz.my", "com.my", - "net.my", - "org.my", - "gov.my", "edu.my", + "gov.my", "mil.my", "name.my", + "net.my", + "org.my", "mz", "ac.mz", "adv.mz", @@ -5100,11 +5103,11 @@ object PublicSuffixes { "org.pe", "com.pe", "net.pe", - "pf", - "com.pf" + "pf" ) private def publicSuffixes1 = Set( + "com.pf", "org.pf", "edu.pf", "ph", @@ -6128,6 +6131,7 @@ object PublicSuffixes { "հայ", "বাংলা", "бг", + "البحرين", "бел", "中国", "中國", @@ -6166,6 +6170,7 @@ object PublicSuffixes { "الاردن", "한국", "қаз", + "ລາວ", "ලංකා", "இலங்கை", "المغرب", @@ -6210,6 +6215,13 @@ object PublicSuffixes { "укр", "اليمن", "xxx", + "ye", + "com.ye", + "edu.ye", + "gov.ye", + "net.ye", + "mil.ye", + "org.ye", "ac.za", "agric.za", "alt.za", @@ -6418,7 +6430,6 @@ object PublicSuffixes { "cars", "casa", "case", - "caseih", "cash", "casino", "catering", @@ -6427,7 +6438,6 @@ object PublicSuffixes { "cbn", "cbre", "cbs", - "ceb", "center", "ceo", "cern", @@ -6759,7 +6769,6 @@ object PublicSuffixes { "jaguar", "java", "jcb", - "jcp", "jeep", "jetzt", "jewelry", @@ -6850,7 +6859,6 @@ object PublicSuffixes { "ltd", "ltda", "lundbeck", - "lupin", "luxe", "luxury", "macys", @@ -6920,7 +6928,6 @@ object PublicSuffixes { "network", "neustar", "new", - "newholland", "news", "next", "nextdirect", @@ -7128,7 +7135,6 @@ object PublicSuffixes { "shouji", "show", "showtime", - "shriram", "silk", "sina", "singles", @@ -7456,6 +7462,7 @@ object PublicSuffixes { "us-gov-west-1.elasticbeanstalk.com", "us-west-1.elasticbeanstalk.com", "us-west-2.elasticbeanstalk.com", + "awsglobalaccelerator.com", "s3.amazonaws.com", "s3-ap-northeast-1.amazonaws.com", "s3-ap-northeast-2.amazonaws.com", @@ -7514,6 +7521,8 @@ object PublicSuffixes { "t3l3p0rt.net", "tele.amune.org", "apigee.io", + "appspacehosted.com", + "appspaceusercontent.com", "on-aptible.com", "user.aseinet.ne.jp", "gv.vc", @@ -7532,6 +7541,7 @@ object PublicSuffixes { "betainabox.com", "bnr.la", "blackbaudcdn.net", + "of.je", "boomla.net", "boxfuse.io", "square7.ch", @@ -7589,12 +7599,12 @@ object PublicSuffixes { "shop.ro", "c.la", "certmgr.org", - "xenapponazure.com", "discourse.group", "discourse.team", "virtueeldomein.nl", "cleverapps.io", "clic2000.net", + "clickrising.net", "c66.me", "cloud66.ws", "cloud66.zone", @@ -7628,7 +7638,6 @@ object PublicSuffixes { "cloudns.pro", "cloudns.pw", "cloudns.us", - "cloudeity.net", "cnpy.gdn", "co.nl", "co.no", @@ -7674,12 +7683,15 @@ object PublicSuffixes { "builtwithdark.com", "edgestack.me", "debian.net", + "deno.dev", + "deno-staging.dev", "dedyn.io", "jozi.biz", "dnshome.de", "online.th", "shop.th", "drayddns.com", + "shoparena.pl", "dreamhosters.com", "mydrobo.com", "drud.io", @@ -7979,6 +7991,7 @@ object PublicSuffixes { "ddnss.org", "definima.net", "definima.io", + "ondigitalocean.app", "bci.dnstrace.pro", "ddnsfree.com", "ddnsgeek.com", @@ -8062,6 +8075,7 @@ object PublicSuffixes { "tr.eu.org", "uk.eu.org", "us.eu.org", + "eurodir.ru", "eu-1.evennode.com", "eu-2.evennode.com", "eu-3.evennode.com", @@ -8151,6 +8165,7 @@ object PublicSuffixes { "vologda.su", "channelsdvr.net", "u.channelsdvr.net", + "edgecompute.app", "fastly-terrarium.com", "fastlylb.net", "map.fastlylb.net", @@ -8166,7 +8181,6 @@ object PublicSuffixes { "myfast.host", "fastvps.site", "myfast.space", - "fhapp.xyz", "fedorainfracloud.org", "fedorapeople.org", "cloud.fedoraproject.org", @@ -8177,6 +8191,7 @@ object PublicSuffixes { "couk.me", "ukco.me", "mydobiss.com", + "fh-muenster.io", "filegear.me", "filegear-au.me", "filegear-de.me", @@ -8185,10 +8200,14 @@ object PublicSuffixes { "filegear-jp.me", "filegear-sg.me", "firebaseapp.com", + "fireweb.app", + "flap.id", "fly.dev", "edgeapp.net", "shw.io", "flynnhosting.net", + "framer.app", + "framercanvas.com", "0e.vc", "freebox-os.com", "freeboxos.com", @@ -8197,6 +8216,7 @@ object PublicSuffixes { "freebox-os.fr", "freeboxos.fr", "freedesktop.org", + "freemyip.com", "wien.funkfeuer.at", "futurehosting.at", "futuremailing.at", @@ -8207,6 +8227,7 @@ object PublicSuffixes { "gentlentapis.com", "lab.ms", "cdn-edges.net", + "ghost.io", "github.io", "githubusercontent.com", "gitlab.io", @@ -8310,6 +8331,8 @@ object PublicSuffixes { "blogspot.vn", "graphox.us", "awsmppl.com", + "günstigbestellen.de", + "günstigliefern.de", "fin.ci", "free.hr", "caa.li", @@ -8328,22 +8351,26 @@ object PublicSuffixes { "ravendb.me", "development.run", "ravendb.run", - "bpl.biz", + "secaas.hk", "orx.biz", - "ng.city", "biz.gl", - "ng.ink", "col.ng", "firm.ng", "gen.ng", "ltd.ng", "ngo.ng", - "ng.school", + "edu.scot", "sch.so", + "org.yt", "hostyhosting.io", "häkkinen.fi", "moonscale.net", "iki.fi", + "smushcdn.com", + "wphostedmail.com", + "wpmucdn.com", + "tempurl.host", + "wpmudev.host", "dyn-berlin.de", "in-berlin.de", "in-brb.de", @@ -8406,6 +8433,9 @@ object PublicSuffixes { "it1.eur.aruba.jenv-aruba.cloud", "it1.jenv-aruba.cloud", "it1-eur.jenv-arubabiz.cloud", + "oxa.cloud", + "tn.oxa.cloud", + "uk.oxa.cloud", "primetel.cloud", "uk.primetel.cloud", "ca.reclaim.cloud", @@ -8476,6 +8506,7 @@ object PublicSuffixes { "enscaled.sg", "jele.site", "jelastic.team", + "orangecloud.tn", "j.layershift.co.uk", "phx.enscaled.us", "mircloud.us", @@ -8488,8 +8519,12 @@ object PublicSuffixes { "uni5.net", "knightpoint.systems", "oya.to", + "kuleuven.cloud", + "ezproxy.kuleuven.be", "co.krd", "edu.krd", + "krellian.net", + "webthings.io", "git-repos.de", "lcube-server.de", "svn-repos.de", @@ -8505,7 +8540,6 @@ object PublicSuffixes { "co.place", "co.technology", "app.lmpm.com", - "linkitools.space", "linkyard.cloud", "linkyard-cloud.ch", "members.linode.com", @@ -8516,13 +8550,13 @@ object PublicSuffixes { "loginline.io", "loginline.services", "loginline.site", + "lohmus.me", "krasnik.pl", "leczna.pl", "lubartow.pl", "lublin.pl", "poniatowa.pl", "swidnik.pl", - "uklugs.org", "glug.org.uk", "lug.org.uk", "lugs.org.uk", @@ -8553,7 +8587,9 @@ object PublicSuffixes { "mayfirst.org", "hb.cldmail.ru", "mcpe.me", + "mcdir.me", "mcdir.ru", + "mcpre.ru", "vps.mcdir.ru", "miniserver.com", "memset.net", @@ -8566,6 +8602,12 @@ object PublicSuffixes { "azurewebsites.net", "azure-mobile.net", "cloudapp.net", + "azurestaticapps.net", + "centralus.azurestaticapps.net", + "eastasia.azurestaticapps.net", + "eastus2.azurestaticapps.net", + "westeurope.azurestaticapps.net", + "westus2.azurestaticapps.net", "csx.cc", "forte.id", "mozilla-iot.org", @@ -8575,8 +8617,11 @@ object PublicSuffixes { "pp.ru", "hostedpi.com", "customer.mythic-beasts.com", + "caracal.mythic-beasts.com", + "fentiger.mythic-beasts.com", "lynx.mythic-beasts.com", "ocelot.mythic-beasts.com", + "oncilla.mythic-beasts.com", "onza.mythic-beasts.com", "sphinx.mythic-beasts.com", "vs.mythic-beasts.com", @@ -8586,16 +8631,19 @@ object PublicSuffixes { "ui.nabu.casa", "pony.club", "of.fashion", - "on.fashion", - "of.football", "in.london", "of.london", + "from.marketing", + "with.marketing", "for.men", + "repair.men", "and.mom", "for.mom", "for.one", + "under.one", "for.sale", - "of.work", + "that.win", + "from.work", "to.work", "nctu.me", "netlify.app", @@ -8603,6 +8651,7 @@ object PublicSuffixes { "ngrok.io", "nh-serv.co.uk", "nfshost.com", + "noticeable.news", "dnsking.ch", "mypi.co", "n4t.co", @@ -8779,6 +8828,7 @@ object PublicSuffixes { "nid.io", "opensocial.site", "opencraft.hosting", + "orsites.com", "operaunite.com", "skygearapp.com", "outsystemscloud.com", @@ -8803,6 +8853,9 @@ object PublicSuffixes { "gotpantheon.com", "mypep.link", "perspecta.cloud", + "lk3.ru", + "ra-ru.ru", + "zsew.ru", "on-web.fr", "bc.platform.sh", "ent.platform.sh", @@ -8823,8 +8876,10 @@ object PublicSuffixes { "chirurgiens-dentistes-en-france.fr", "byen.site", "pubtls.org", + "qoto.io", "qualifioapp.com", "qbuser.com", + "cloudsite.builders", "instantcloud.cn", "ras.ru", "qa2.com", @@ -8847,7 +8902,6 @@ object PublicSuffixes { "devices.resinstaging.io", "hzc.io", "wellbeingzone.eu", - "ptplus.fit", "wellbeingzone.co.uk", "git-pages.rit.edu", "sandcats.io", @@ -8855,6 +8909,7 @@ object PublicSuffixes { "logoip.com", "schokokeks.net", "gov.scot", + "service.gov.scot", "scrysec.com", "firewall-gateway.com", "firewall-gateway.de", @@ -8873,6 +8928,7 @@ object PublicSuffixes { "pp.ua", "shiftedit.io", "myshopblocks.com", + "myshopify.com", "shopitsite.com", "shopware.store", "mo-siemens.io", @@ -8886,6 +8942,8 @@ object PublicSuffixes { "alpha.bounty-full.com", "beta.bounty-full.com", "small-web.org", + "try-snowplow.com", + "srht.site", "stackhero-network.com", "static.land", "dev.static.land", @@ -8900,8 +8958,6 @@ object PublicSuffixes { "soc.srcf.net", "user.srcf.net", "temp-dns.com", - "applicationcloud.io", - "scapp.io", "syncloud.it", "diskstation.me", "dscloud.biz", @@ -8936,6 +8992,8 @@ object PublicSuffixes { "arvo.network", "azimuth.network", "tlon.network", + "torproject.net", + "pages.torproject.net", "bloxcms.com", "townnews-staging.com", "12hp.at", @@ -9032,7 +9090,7 @@ object PublicSuffixes { "wafflecell.com", "idnblogger.com", "indowapblog.com", - "bloghp.id", + "bloger.id", "wblog.id", "wbq.me", "fastblog.net", @@ -9046,6 +9104,7 @@ object PublicSuffixes { "wmcloud.org", "panel.gg", "daemon.panel.gg", + "woltlab-demo.com", "myforum.community", "community-pro.de", "diskussionsbereich.de", @@ -9085,6 +9144,16 @@ object PublicSuffixes { "js.wpenginepowered.com", "impertrixcdn.com", "impertrix.com", - "gsj.bz" + "gsj.bz", + "биз.рус", + "ком.рус", + "крым.рус", + "мир.рус", + "мск.рус", + "орг.рус", + "самара.рус", + "сочи.рус", + "спб.рус", + "я.рус" ) } diff --git a/shared/src/main/scala/io/lemonlabs/uri/redact/Redact.scala b/shared/src/main/scala/io/lemonlabs/uri/redact/Redact.scala new file mode 100644 index 00000000..e5fedb6d --- /dev/null +++ b/shared/src/main/scala/io/lemonlabs/uri/redact/Redact.scala @@ -0,0 +1,46 @@ +package io.lemonlabs.uri.redact + +import io.lemonlabs.uri.Url +import io.lemonlabs.uri.typesafe.QueryKey + +import QueryKey.ops._ + +object Redact { + def byRemoving: RedactByRemoving = RedactByRemoving(identity) + def withPlaceholder(placeholder: String): RedactByReplacing = RedactByReplacing(placeholder, identity) +} + +trait Redactor { + def apply(u: Url): Url +} + +case class RedactByRemoving(f: Url => Url) extends Redactor { + + private def andThen(next: Url => Url) = copy(f = f.andThen(next)) + + def allParams(): RedactByRemoving = andThen(_.removeQueryString()) + def params[K: QueryKey](names: K*): RedactByRemoving = andThen(_.removeParams(names)) + def userInfo(): RedactByRemoving = andThen(_.removeUserInfo()) + def password(): RedactByRemoving = andThen(_.removePassword()) + + def apply(u: Url): Url = f(u) +} + +case class RedactByReplacing(placeholder: String, f: Url => Url) extends Redactor { + + private def andThen(next: Url => Url) = copy(f = f.andThen(next)) + + def allParams(): RedactByReplacing = andThen(_.mapQueryValues(_ => placeholder)) + def params[K: QueryKey](names: K*): RedactByReplacing = { + val namesStr = names.map(_.queryKey) + andThen { url => + url.mapQuery { + case (k, _) if namesStr.contains(k) => k -> placeholder + } + } + } + def user(): RedactByReplacing = andThen(_.mapUser(_ => placeholder)) + def password(): RedactByReplacing = andThen(_.mapPassword(_ => placeholder)) + + def apply(u: Url): Url = f(u) +} diff --git a/shared/src/test/scala/io/lemonlabs/uri/RedactTests.scala b/shared/src/test/scala/io/lemonlabs/uri/RedactTests.scala new file mode 100644 index 00000000..ee1e6c89 --- /dev/null +++ b/shared/src/test/scala/io/lemonlabs/uri/RedactTests.scala @@ -0,0 +1,64 @@ +package io.lemonlabs.uri + +import io.lemonlabs.uri.redact.Redact +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +class RedactTests extends AnyWordSpec with Matchers { + "Redacting byRemoving" should { + "remove all parameters" in { + val url = Url.parse("http://user:password@example.com?secret=123&other=true") + url.toRedactedString(Redact.byRemoving.allParams()) should equal("http://user:password@example.com") + } + "remove parameters by name" in { + val url = Url.parse("http://user:password@example.com?secret=123&other=true&other=false&last=yes") + url.toRedactedString(Redact.byRemoving.params("secret", "other")) should equal( + "http://user:password@example.com?last=yes" + ) + } + "remove user info" in { + val url = Url.parse("http://user:password@example.com?secret=123&other=true") + url.toRedactedString(Redact.byRemoving.userInfo()) should equal("http://example.com?secret=123&other=true") + } + "remove password" in { + val url = Url.parse("http://user:password@example.com?secret=123&other=true") + url.toRedactedString(Redact.byRemoving.password()) should equal("http://user@example.com?secret=123&other=true") + } + "remove everything" in { + val url = Url.parse("http://user:password@example.com?secret=123&other=true") + url.toRedactedString(Redact.byRemoving.allParams().userInfo()) should equal("http://example.com") + } + } + "Redacting withPlaceholder" should { + "replace all parameters" in { + val url = Url.parse("http://user:password@example.com?secret=123&other=true") + url.toRedactedString(Redact.withPlaceholder("xxx").allParams()) should equal( + "http://user:password@example.com?secret=xxx&other=xxx" + ) + } + "replace parameters by name" in { + val url = Url.parse("http://user:password@example.com?secret=123&other=true&other=false&last=yes") + url.toRedactedString(Redact.withPlaceholder("xxx").params("secret", "other")) should equal( + "http://user:password@example.com?secret=xxx&other=xxx&other=xxx&last=yes" + ) + } + "replace user" in { + val url = Url.parse("http://user:password@example.com?secret=123&other=true") + url.toRedactedString(Redact.withPlaceholder("xxx").user()) should equal( + "http://xxx:password@example.com?secret=123&other=true" + ) + } + "replace password" in { + val url = Url.parse("http://user:password@example.com?secret=123&other=true") + url.toRedactedString(Redact.withPlaceholder("xxx").password()) should equal( + "http://user:xxx@example.com?secret=123&other=true" + ) + } + "replace everything" in { + val url = Url.parse("http://user:password@example.com?secret=123&other=true") + url.toRedactedString(Redact.withPlaceholder("xxx").allParams().user().password()) should equal( + "http://xxx:xxx@example.com?secret=xxx&other=xxx" + ) + } + } +} diff --git a/shared/src/test/scala/io/lemonlabs/uri/TransformTests.scala b/shared/src/test/scala/io/lemonlabs/uri/TransformTests.scala index 5e102722..d27792f4 100644 --- a/shared/src/test/scala/io/lemonlabs/uri/TransformTests.scala +++ b/shared/src/test/scala/io/lemonlabs/uri/TransformTests.scala @@ -144,4 +144,95 @@ class TransformTests extends AnyWordSpec with Matchers { uri2.toString should equal("/test?param_1=hello¶m_1=hello¶m_2=goodbye¶m_2=goodbye") } } + + "mapUser" should { + "not change a relative URL" in { + val uri = Url.parse("/test?param_1=hello¶m_2=goodbye") + uri.mapUser(_ + "2") should equal(uri) + } + "not change a URL without authority" in { + val uri = Url.parse("mailto:me@example.com") + uri.mapUser(_ + "2") should equal(uri) + } + "change the user in an absolute URL" in { + val uri = Url.parse("http://me@example.com/test").mapUser(_ + "2") + uri.user should equal(Some("me2")) + uri.toString should equal("http://me2@example.com/test") + } + "not change an absolute URL with no user-info" in { + val uri = Url.parse("http://example.com/test") + uri.mapUser(_ + "2") should equal(uri) + } + } + + "mapPassword" should { + "not change a relative URL" in { + val uri = Url.parse("/test?param_1=hello¶m_2=goodbye") + uri.mapPassword(_ + "2") should equal(uri) + } + "not change a URL without authority" in { + val uri = Url.parse("mailto:me@example.com") + uri.mapPassword(_ + "2") should equal(uri) + } + "change the password in an absolute URL" in { + val uri = Url.parse("http://me:password@example.com/test").mapPassword(_ + "2") + uri.password should equal(Some("password2")) + uri.toString should equal("http://me:password2@example.com/test") + } + "not change an absolute URL with no user-info" in { + val uri = Url.parse("http://example.com/test") + uri.mapPassword(_ + "2") should equal(uri) + } + } + + "removeUserInfo" should { + "not change a relative URL" in { + val uri = Url.parse("/test?param_1=hello¶m_2=goodbye") + uri.removeUserInfo() should equal(uri) + } + "not change a URL without authority" in { + val uri = Url.parse("mailto:me@example.com") + uri.removeUserInfo() should equal(uri) + } + "remove a user and password in an absolute URL" in { + val uri = Url.parse("http://me:password@example.com/test").removeUserInfo() + uri.user should equal(None) + uri.password should equal(None) + uri.toString should equal("http://example.com/test") + } + "remove a user from an absolute URL" in { + val uri = Url.parse("http://me@example.com/test").removeUserInfo() + uri.user should equal(None) + uri.password should equal(None) + uri.toString should equal("http://example.com/test") + } + "not change an absolute URL with no user-info" in { + val uri = Url.parse("http://example.com/test") + uri.removeUserInfo() should equal(uri) + } + } + + "removePassword" should { + "not change a relative URL" in { + val uri = Url.parse("/test?param_1=hello¶m_2=goodbye") + uri.removePassword() should equal(uri) + } + "not change a URL without authority" in { + val uri = Url.parse("mailto:me@example.com") + uri.removePassword() should equal(uri) + } + "remove the password in an absolute URL" in { + val uri = Url.parse("http://me:password@example.com/test").removePassword() + uri.password should equal(None) + uri.toString should equal("http://me@example.com/test") + } + "not change an absolute URL without a password" in { + val uri = Url.parse("http://me@example.com/test") + uri.removePassword() should equal(uri) + } + "not change an absolute URL with no user-info" in { + val uri = Url.parse("http://example.com/test") + uri.removePassword() should equal(uri) + } + } } diff --git a/version.sbt b/version.sbt index a7c0bf66..3271bb8f 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "3.0.0" +version in ThisBuild := "3.1.0"