Skip to content

Commit

Permalink
Upgraded to spark-3.3
Browse files Browse the repository at this point in the history
  • Loading branch information
sharath-sg2706 committed Oct 17, 2022
1 parent 053f150 commit 99e769f
Show file tree
Hide file tree
Showing 5 changed files with 57 additions and 45 deletions.
1 change: 0 additions & 1 deletion .bsp/sbt.json

This file was deleted.

47 changes: 47 additions & 0 deletions .github/workflows/http.almaren-githubactions.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
name: HTTP.Almaren
on: [push, pull_request]

jobs:
Build:
runs-on: ubuntu-20.04
services:
postgres:
image: postgres:13.4
env:
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: trust
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name : Check out repository code
uses: actions/checkout@v2
- name: Setup JDK
uses: actions/setup-java@v3
with:
distribution: temurin
java-version: 8
cache: sbt
- name: Setup web environment
run: |
cpanm --local-lib=~/perl5 local::lib && eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib)
cpanm Mojolicious
cpanm JSON::Parse
perl src/test/resources/script/mock_api.pl daemon -m dev -l http://\*:3000 &
- name: Build and test scala version
run: |
PGPASSWORD="postgres" psql -c 'create database almaren;' -U postgres -h localhost
PGPASSWORD="postgres" psql -c "ALTER USER postgres PASSWORD 'foo' ;" -U postgres -h localhost
PGPASSWORD="postgres" psql -c 'create role runner;' -U postgres -h localhost
PGPASSWORD="postgres" psql -c 'ALTER ROLE "runner" WITH LOGIN SUPERUSER INHERIT CREATEDB CREATEROLE REPLICATION;' -U postgres -h localhost
sbt ++2.12.15 test
sbt ++2.13.9 test
rm -rf "$HOME/.ivy2/local" || true
find $HOME/Library/Caches/Coursier/v1 -name "ivydata-*.properties" -delete || true
find $HOME/.ivy2/cache -name "ivydata-*.properties" -delete || true
find $HOME/.cache/coursier/v1 -name "ivydata-*.properties" -delete || true
find $HOME/.sbt -name "*.lock" -delete || true
37 changes: 0 additions & 37 deletions .travis.yml

This file was deleted.

14 changes: 8 additions & 6 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ ThisBuild / name := "http.almaren"
ThisBuild / organization := "com.github.music-of-the-ainur"

lazy val scala212 = "2.12.15"
lazy val scala213 = "2.13.9"

ThisBuild / scalaVersion := scala212
crossScalaVersions := Seq(scala212,scala213)
ThisBuild / scalaVersion := scala213

val sparkVersion = "3.2.1"
val sparkVersion = "3.3.0"
val majorVersionReg = "([0-9]+\\.[0-9]+).{0,}".r

val majorVersionReg(majorVersion) = sparkVersion
Expand All @@ -15,10 +17,10 @@ scalacOptions ++= Seq("-deprecation", "-feature")
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
"com.github.music-of-the-ainur" %% "almaren-framework" % s"0.9.4-${majorVersion}" % "provided",
"com.lihaoyi" %% "requests" % "0.7.0",
"com.typesafe.scala-logging" %% "scala-logging" % "3.9.0",
"org.scalatest" %% "scalatest" % "3.0.5" % "test"
"com.github.music-of-the-ainur" %% "almaren-framework" % s"0.9.8-${majorVersion}" % "provided",
"com.lihaoyi" %% "requests" % "0.7.1",
"com.typesafe.scala-logging" %% "scala-logging" % "3.9.5",
"org.scalatest" %% "scalatest" % "3.2.14" % "test"
)

enablePlugins(GitVersioning)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ import com.github.music.of.the.ainur.almaren.Almaren
import com.github.music.of.the.ainur.almaren.builder.Core.Implicit
import com.github.music.of.the.ainur.almaren.http.HTTPConn.HTTPImplicit
import org.apache.spark.sql.Row
import org.scalatest.funsuite.AnyFunSuite

class Test extends FunSuite with BeforeAndAfter {
class Test extends AnyFunSuite with BeforeAndAfter {
val almaren = Almaren("http-almaren")

val spark: SparkSession = almaren.spark
Expand Down

0 comments on commit 99e769f

Please sign in to comment.