Bidder Configuration Fixes & URL Validation

extra/modules/live-intent-omni-channel-identity/src/main/java/org/prebid/server/hooks/modules/liveintent/omni/channel/identity/v1/hooks/LiveIntentOmniChannelIdentityProcessedAuctionRequestHook.java

@@ -60,7 +60,7 @@ public LiveIntentOmniChannelIdentityProcessedAuctionRequestHook(LiveIntentOmniCh
                                                                     double logSamplingRate) {
 
         this.config = Objects.requireNonNull(config);
-        HttpUtil.validateUrlSyntax(config.getIdentityResolutionEndpoint());
+        HttpUtil.validateUrl(config.getIdentityResolutionEndpoint());
         this.mapper = Objects.requireNonNull(mapper);
         this.httpClient = Objects.requireNonNull(httpClient);
         this.logSamplingRate = logSamplingRate;

extra/modules/wurfl-devicedetection/src/main/java/org/prebid/server/hooks/modules/com/scientiamobile/wurfl/devicedetection/model/WURFLEngineUtils.java

@@ -4,6 +4,7 @@
 import com.scientiamobile.wurfl.core.WURFLEngine;
 import com.scientiamobile.wurfl.core.cache.LRUMapCacheProvider;
 import com.scientiamobile.wurfl.core.cache.NullCacheProvider;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.hooks.modules.com.scientiamobile.wurfl.devicedetection.config.WURFLDeviceDetectionConfigProperties;
 import org.prebid.server.hooks.modules.com.scientiamobile.wurfl.devicedetection.exc.WURFLDeviceDetectionException;
 
@@ -56,7 +57,7 @@ private static String wurflFilePathFromConfig(WURFLDeviceDetectionConfigProperti
 
     public static String extractWURFLFileName(String wurflSnapshotUrl) {
         try {
-            final URI uri = new URI(wurflSnapshotUrl);
+            final URI uri = new URIBuilder(wurflSnapshotUrl).build();
             final String path = uri.getPath();
             return path.substring(path.lastIndexOf('/') + 1);
         } catch (Exception e) {

extra/pom.xml

@@ -39,9 +39,7 @@
         <commons.collections.version>4.4</commons.collections.version>
         <commons.compress.version>1.27.1</commons.compress.version>
         <commons-math3.version>3.6.1</commons-math3.version>
-        <commons-validator.version>1.10.0</commons-validator.version>
         <scram.version>2.1</scram.version>
-        <httpclient.version>4.5.14</httpclient.version>
         <ipaddress.version>5.5.1</ipaddress.version>
         <oshi.version>6.8.0</oshi.version>
         <json-schema-validator.version>1.5.6</json-schema-validator.version>
@@ -136,23 +134,6 @@
                 <artifactId>commons-math3</artifactId>
                 <version>${commons-math3.version}</version>
             </dependency>
-            <dependency>
-                <groupId>commons-validator</groupId>
-                <artifactId>commons-validator</artifactId>
-                <version>${commons-validator.version}</version>
-            </dependency>
-            <!-- TODO: refactor code to replace URIBuilder with something else so that this dep can be removed -->
-            <dependency>
-                <groupId>org.apache.httpcomponents</groupId>
-                <artifactId>httpclient</artifactId>
-                <version>${httpclient.version}</version>
-                <exclusions>
-                    <exclusion>
-                        <groupId>commons-logging</groupId>
-                        <artifactId>commons-logging</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
             <dependency>
                 <groupId>com.github.seancfoley</groupId>
                 <artifactId>ipaddress</artifactId>

pom.xml

@@ -111,12 +111,8 @@
             <artifactId>commons-math3</artifactId>
         </dependency>
         <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpclient</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>commons-validator</groupId>
-            <artifactId>commons-validator</artifactId>
+            <groupId>org.apache.httpcomponents.client5</groupId>
+            <artifactId>httpclient5</artifactId>
         </dependency>
         <dependency>
             <groupId>com.github.seancfoley</groupId>

src/main/java/org/prebid/server/auction/IpAddressHelper.java

@@ -5,7 +5,7 @@
 import inet.ipaddr.IPAddressString;
 import inet.ipaddr.IPAddressStringParameters;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.conn.util.InetAddressUtils;
+import org.apache.hc.core5.net.InetAddressUtils;
 import org.prebid.server.auction.model.IpAddress;
 import org.prebid.server.log.Logger;
 import org.prebid.server.log.LoggerFactory;

src/main/java/org/prebid/server/bidder/adgeneration/AdgenerationBidder.java

@@ -14,7 +14,7 @@
 import io.vertx.core.http.HttpMethod;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.adgeneration.model.AdgenerationResponse;
 import org.prebid.server.bidder.model.BidderBid;

src/main/java/org/prebid/server/bidder/adkernel/AdkernelBidder.java

@@ -41,6 +41,8 @@ public class AdkernelBidder implements Bidder<BidRequest> {
             new TypeReference<>() {
             };
 
+    private static final String ZONE_ID_MACRO = "{{ZoneId}}";
+
     private static final String MF_SUFFIX = "__mf";
     private static final String MF_SUFFIX_BANNER = "b" + MF_SUFFIX;
     private static final String MF_SUFFIX_VIDEO = "v" + MF_SUFFIX;
@@ -49,11 +51,11 @@ public class AdkernelBidder implements Bidder<BidRequest> {
 
     private static final int MF_SUFFIX_LENGTH = MF_SUFFIX.length() + 1;
 
-    private final String endpointTemplate;
+    private final String endpointUrl;
     private final JacksonMapper mapper;
 
-    public AdkernelBidder(String endpointTemplate, JacksonMapper mapper) {
-        this.endpointTemplate = HttpUtil.validateUrl(Objects.requireNonNull(endpointTemplate));
+    public AdkernelBidder(String endpointUrl, JacksonMapper mapper) {
+        this.endpointUrl = HttpUtil.validateUrl(Objects.requireNonNull(endpointUrl));
         this.mapper = Objects.requireNonNull(mapper);
     }
 
@@ -183,10 +185,9 @@ private HttpRequest<BidRequest> createHttpRequest(Map.Entry<ExtImpAdkernel, List
                                                       App app) {
 
         final ExtImpAdkernel impExt = extAndImp.getKey();
-        final String uri = endpointTemplate.formatted(impExt.getZoneId());
+        final String uri = endpointUrl.replace(ZONE_ID_MACRO, HttpUtil.encodeUrl(impExt.getZoneId().toString()));
 
-        final MultiMap headers = HttpUtil.headers()
-                .add(HttpUtil.X_OPENRTB_VERSION_HEADER, "2.5");
+        final MultiMap headers = HttpUtil.headers().add(HttpUtil.X_OPENRTB_VERSION_HEADER, "2.5");
 
         final BidRequest outgoingRequest = createBidRequest(extAndImp.getValue(), requestBuilder, site, app);
 

src/main/java/org/prebid/server/bidder/adnuntius/AdnuntiusBidder.java

@@ -22,7 +22,7 @@
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.adnuntius.model.request.AdnuntiusMetaData;
 import org.prebid.server.bidder.adnuntius.model.request.AdnuntiusNativeRequest;

src/main/java/org/prebid/server/bidder/adocean/AdoceanBidder.java

@@ -16,10 +16,9 @@
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.math.NumberUtils;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.client.utils.URLEncodedUtils;
-import org.apache.http.message.BasicNameValuePair;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.http.message.BasicNameValuePair;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.adocean.model.AdoceanResponseAdUnit;
 import org.prebid.server.bidder.model.BidderBid;
@@ -37,9 +36,7 @@
 
 import java.io.IOException;
 import java.math.BigDecimal;
-import java.net.URI;
 import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -300,7 +297,7 @@ private static MultiMap getHeaders(BidRequest request) {
     public Result<List<BidderBid>> makeBids(BidderCall<Void> httpCall, BidRequest bidRequest) {
         final List<NameValuePair> params;
         try {
-            params = URLEncodedUtils.parse(new URI(httpCall.getRequest().getUri()), StandardCharsets.UTF_8);
+            params = new URIBuilder(httpCall.getRequest().getUri()).getQueryParams();
         } catch (URISyntaxException e) {
             return Result.withError(BidderError.badInput(e.getMessage()));
         }

src/main/java/org/prebid/server/bidder/amx/AmxBidder.java

@@ -13,7 +13,7 @@
 import com.iab.openrtb.response.SeatBid;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.amx.model.AmxBidExt;
 import org.prebid.server.bidder.model.BidderBid;

src/main/java/org/prebid/server/bidder/appnexus/AppnexusBidder.java

@@ -19,7 +19,7 @@
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.auction.model.Endpoint;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.appnexus.proto.AppnexusBidExt;

src/main/java/org/prebid/server/bidder/audiencenetwork/AudienceNetworkBidder.java

@@ -53,23 +53,26 @@ public class AudienceNetworkBidder implements Bidder<BidRequest> {
 
     private static final List<Integer> SUPPORTED_BANNER_HEIGHT = Arrays.asList(250, 50);
 
+    private static final String PLATFORM_MACRO = "{{PlatformId}}";
+    private static final String REQUEST_MACRO = "{{RequestId}}";
+    private static final String PUBLISHER_MACRO = "{{PublisherId}}";
+
     private final String endpointUrl;
     private final String platformId;
     private final String appSecret;
-    private final String timeoutNotificationUrlTemplate;
+    private final String timeoutNotificationUrl;
     private final JacksonMapper mapper;
 
     public AudienceNetworkBidder(String endpointUrl,
                                  String platformId,
                                  String appSecret,
-                                 String timeoutNotificationUrlTemplate,
+                                 String timeoutNotificationUrl,
                                  JacksonMapper mapper) {
 
         this.endpointUrl = HttpUtil.validateUrl(Objects.requireNonNull(endpointUrl));
         this.platformId = checkBlankString(Objects.requireNonNull(platformId), "platform-id");
         this.appSecret = checkBlankString(Objects.requireNonNull(appSecret), "app-secret");
-        this.timeoutNotificationUrlTemplate = HttpUtil.validateUrl(
-                Objects.requireNonNull(timeoutNotificationUrlTemplate));
+        this.timeoutNotificationUrl = HttpUtil.validateUrl(Objects.requireNonNull(timeoutNotificationUrl));
         this.mapper = Objects.requireNonNull(mapper);
     }
 
@@ -343,7 +346,9 @@ public HttpRequest<Void> makeTimeoutNotification(HttpRequest<BidRequest> httpReq
 
         return HttpRequest.<Void>builder()
                 .method(HttpMethod.GET)
-                .uri(timeoutNotificationUrlTemplate.formatted(platformId, publisherId, requestId))
+                .uri(timeoutNotificationUrl.replace(PLATFORM_MACRO, HttpUtil.encodeUrl(platformId))
+                        .replace(PUBLISHER_MACRO, HttpUtil.encodeUrl(publisherId))
+                        .replace(REQUEST_MACRO, HttpUtil.encodeUrl(requestId)))
                 .build();
     }
 }

src/main/java/org/prebid/server/bidder/bluesea/BlueSeaBidder.java

@@ -9,7 +9,7 @@
 import com.iab.openrtb.response.BidResponse;
 import com.iab.openrtb.response.SeatBid;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.http.client.utils.URIBuilder;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.model.BidderBid;
 import org.prebid.server.bidder.model.BidderCall;

src/main/java/org/prebid/server/bidder/connatix/ConnatixBidder.java

@@ -17,7 +17,7 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.model.BidderBid;
 import org.prebid.server.bidder.model.BidderCall;

src/main/java/org/prebid/server/bidder/dxkulture/DxKultureBidder.java

@@ -10,7 +10,7 @@
 import com.iab.openrtb.response.SeatBid;
 import io.vertx.core.MultiMap;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.http.client.utils.URIBuilder;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.model.BidderBid;
 import org.prebid.server.bidder.model.BidderCall;

src/main/java/org/prebid/server/bidder/emxdigital/EmxDigitalBidder.java

@@ -31,7 +31,7 @@
 import org.prebid.server.util.HttpUtil;
 
 import java.math.BigDecimal;
-import java.time.Instant;
+import java.time.Clock;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -41,16 +41,20 @@ public class EmxDigitalBidder implements Bidder<BidRequest> {
 
     private static final String USD_CURRENCY = "USD";
     private static final Integer PROTOCOL_VAST_40 = 7;
+    private static final String URL_TIMEOUT_MACRO = "{{URL_TIMEOUT}}";
+    private static final String TIMESTAMP_MACRO = "{{TIMESTAMP}}";
 
     private static final TypeReference<ExtPrebid<?, ExtImpEmxDigital>> EMXDIGITAL_EXT_TYPE_REFERENCE =
             new TypeReference<>() {
             };
 
     private final String endpointUrl;
+    private final Clock clock;
     private final JacksonMapper mapper;
 
-    public EmxDigitalBidder(String endpointUrl, JacksonMapper mapper) {
+    public EmxDigitalBidder(String endpointUrl, Clock clock, JacksonMapper mapper) {
         this.endpointUrl = HttpUtil.validateUrl(Objects.requireNonNull(endpointUrl));
+        this.clock = Objects.requireNonNull(clock);
         this.mapper = Objects.requireNonNull(mapper);
     }
 
@@ -231,8 +235,9 @@ private String makeUrl(BidRequest bidRequest) {
         final Long tmax = bidRequest.getTmax();
         final int urlTimeout = tmax == 0 ? 1000 : tmax.intValue();
 
-        return "%s?t=%s&ts=%s&src=pbserver"
-                .formatted(endpointUrl, urlTimeout, (int) Instant.now().getEpochSecond());
+        return endpointUrl
+                .replace(URL_TIMEOUT_MACRO, String.valueOf(urlTimeout))
+                .replace(TIMESTAMP_MACRO, String.valueOf((int) clock.millis()));
     }
 
     @Override

src/main/java/org/prebid/server/bidder/eplanning/EplanningBidder.java

@@ -18,7 +18,7 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.eplanning.model.CleanStepName;
 import org.prebid.server.bidder.eplanning.model.HbResponse;
@@ -286,7 +286,7 @@ private String resolveRequestUri(BidRequest request, List<String> requestsString
 
     private static URL parseUrl(String url) {
         try {
-            return new URL(url);
+            return HttpUtil.parseUrl(url);
         } catch (MalformedURLException e) {
             throw new PreBidException("Invalid url: " + url, e);
         }

src/main/java/org/prebid/server/bidder/flipp/FlippBidder.java

@@ -16,8 +16,8 @@
 import io.vertx.core.http.HttpMethod;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.utils.URLEncodedUtils;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.net.URIBuilder;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.flipp.model.request.CampaignRequestBody;
 import org.prebid.server.bidder.flipp.model.request.CampaignRequestBodyUser;
@@ -47,7 +47,7 @@
 import org.prebid.server.util.HttpUtil;
 import org.prebid.server.util.ObjectUtil;
 
-import java.nio.charset.StandardCharsets;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -168,12 +168,21 @@ private static String resolveContentCode(Site site, ExtImpFlipp extImp) {
                 .map(Site::getPage)
                 .orElse(null);
 
-        return URLEncodedUtils.parse(pageUrl, StandardCharsets.UTF_8)
-                .stream()
-                .filter(nameValuePair -> nameValuePair.getName().contains("flipp-content-code"))
-                .map(NameValuePair::getValue)
-                .findFirst()
-                .orElse(null);
+        if (pageUrl == null) {
+            return null;
+        }
+
+        try {
+            return new URIBuilder(pageUrl)
+                    .getQueryParams()
+                    .stream()
+                    .filter(nameValuePair -> nameValuePair.getName().contains("flipp-content-code"))
+                    .map(NameValuePair::getValue)
+                    .findFirst()
+                    .orElse(null);
+        } catch (URISyntaxException e) {
+            return null;
+        }
     }
 
     private static List<String> resolveKeywords(BidRequest bidRequest) {

src/main/java/org/prebid/server/bidder/missena/MissenaBidder.java

@@ -169,7 +169,7 @@ private MultiMap makeHeaders(Device device, Site site) {
         if (site != null && StringUtils.isNotBlank(site.getPage())) {
             HttpUtil.addHeaderIfValueIsNotEmpty(headers, HttpUtil.REFERER_HEADER, site.getPage());
             try {
-                final URL url = new URL(site.getPage());
+                final URL url = HttpUtil.parseUrl(site.getPage());
                 final String origin = url.getProtocol() + "://" + url.getHost();
                 HttpUtil.addHeaderIfValueIsNotEmpty(headers, HttpUtil.ORIGIN_HEADER, origin);
             } catch (MalformedURLException e) {