Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(debug): redis provider #489

Merged
merged 14 commits into from
Apr 23, 2024
Merged

feat(debug): redis provider #489

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
1d5fad8
feat(debug): redis provider
darkweak Mar 27, 2024
bc938cd
feat(debug): redis provider
darkweak Mar 30, 2024
bbda508
feat(redis): use scan instead of keys
darkweak Apr 2, 2024
fbfbaa8
feat(chore): use Delete single to purge the mapping itself without to…
darkweak Apr 3, 2024
1f073c2
feat(storage): migrate keys to xxhash to reduce cost
darkweak Apr 10, 2024
1e83d8a
feat(redis): introduce ability to use hashtag in the key as prefix
darkweak Apr 11, 2024
08f795d
feat(storage): add ability to hash the key, compress using lz4 values
darkweak Apr 16, 2024
492256e
feat(chore): compress response
darkweak Apr 17, 2024
25215e8
fix(ci): newman and unit tests
darkweak Apr 18, 2024
7c3b8ba
feat(plugins): sync middleware and storage code to upstream
darkweak Apr 19, 2024
04defb0
fix(review): from hussam-almarzoq
darkweak Apr 22, 2024
2efe7c3
feat(tyk): ci lint fixes
darkweak Apr 22, 2024
8935221
feat(tyk): use dc v3 instead of deprecated v3.8
darkweak Apr 22, 2024
9c9fb17
fix(tyk): remove lz4 dep
darkweak Apr 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/non-regression.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ jobs:
- name: Build and run the docker stack
run: |
docker network create your_network || true
docker-compose -f docker-compose.yml.test up -d --build --force-recreate --remove-orphans
docker compose -f docker-compose.yml.test up -d --build --force-recreate --remove-orphans
- name: Run pkg storage tests
run: docker-compose -f docker-compose.yml.test exec -T souin go test -v -race ./pkg/storage
run: docker compose -f docker-compose.yml.test exec -T souin go test -v -race ./pkg/storage
validate-prod-container-building:
needs: unit-test-golang-with-services
name: Validate that the container build for prod
Expand All @@ -65,7 +65,7 @@ jobs:
run: docker network create your_network || true
-
name: Build the stack
run: docker-compose -f docker-compose.yml.prod up -d --build --force-recreate --remove-orphans
run: docker compose -f docker-compose.yml.prod up -d --build --force-recreate --remove-orphans
-
name: Souin container healthceck
run: docker-compose -f docker-compose.yml.prod exec -T souin ls
run: docker compose -f docker-compose.yml.prod exec -T souin ls
15 changes: 15 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ cache_keys:
disable_host: true # Prevent the host from being used in the cache key
disable_method: true # Prevent the method from being used in the cache key
disable_query: true # Prevent the query string from being used in the cache key
disable_scheme: true # request scheme the query string from being used in the cache key
hash: true # Hash the cache key instead of a plaintext one
hide: true # Prevent the cache key to be in the response Cache-Status header
headers: # Add headers to the key
- Authorization # Add the header value in the key
- Content-Type # Add the header value in the key
Expand All @@ -107,6 +110,9 @@ default_cache:
disable_host: true # Prevent the host from being used in the cache key
disable_method: true # Prevent the method from being used in the cache key
disable_query: true # Prevent the query string from being used in the cache key
disable_scheme: true # Prevent the request scheme string from being used in the cache key
hash: true # Hash the cache key instead of a plaintext one
hide: true # Prevent the cache key to be in the response Cache-Status header
headers: # Add headers to the key
- Authorization # Add the header value in the key
- Content-Type # Add the header value in the key
Expand Down Expand Up @@ -172,6 +178,8 @@ surrogate_keys:
| `cache_keys.{your regexp}.disable_host` | Disable the host part in the key matching the regexp | `true`<br/><br/>`(default: false)` |
| `cache_keys.{your regexp}.disable_method` | Disable the method part in the key matching the regexp | `true`<br/><br/>`(default: false)` |
| `cache_keys.{your regexp}.disable_query` | Disable the query string part in the key matching the regexp | `true`<br/><br/>`(default: false)` |
| `cache_keys.{your regexp}.disable_scheme` | Disable the request scheme string part in the key matching the regexp | `true`<br/><br/>`(default: false)` |
| `cache_keys.{your regexp}.hash` | Hash the key matching the regexp | `true`<br/><br/>`(default: false)` |
| `cache_keys.{your regexp}.headers` | Add headers to the key matching the regexp | `- Authorization`<br/><br/>`- Content-Type`<br/><br/>`- X-Additional-Header` |
| `cache_keys.{your regexp}.hide` | Prevent the key from being exposed in the `Cache-Status` HTTP response header | `true`<br/><br/>`(default: false)` |
| `cdn` | The CDN management, if you use any cdn to proxy your requests Souin will handle that | |
Expand All @@ -197,6 +205,8 @@ surrogate_keys:
| `default_cache.key.disable_host` | Disable the host part in the key | `true`<br/><br/>`(default: false)` |
| `default_cache.key.disable_method` | Disable the method part in the key | `true`<br/><br/>`(default: false)` |
| `default_cache.key.disable_query` | Disable the query string part in the key | `true`<br/><br/>`(default: false)` |
| `default_cache.key.disable_scheme` | Disable the request scheme string part in the key | `true`<br/><br/>`(default: false)` |
| `default_cache.key.hash` | Hash the key name in the storage | `true`<br/><br/>`(default: false)` |
| `default_cache.key.headers` | Add headers to the key matching the regexp | `- Authorization`<br/><br/>`- Content-Type`<br/><br/>`- X-Additional-Header` |
| `default_cache.key.hide` | Prevent the key from being exposed in the `Cache-Status` HTTP response header | `true`<br/><br/>`(default: false)` |
| `default_cache.mode` | RFC respect tweaking | One of `bypass` `bypass_request` `bypass_response` `strict` (default `strict`) |
Expand Down Expand Up @@ -468,8 +478,10 @@ There is the fully configuration below
disable_host
disable_method
disable_query
disable_scheme
headers X-Token Authorization
hide
hash
}
}
cdn {
Expand All @@ -488,6 +500,9 @@ There is the fully configuration below
disable_host
disable_method
disable_query
disable_scheme
hash
hide
headers Content-Type Authorization
}
log_level debug
Expand Down
8 changes: 8 additions & 0 deletions configurationtypes/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,12 @@ func (c *CacheKeys) parseJSON(rootDecoder *json.Decoder) {
case "disable_query":
val, _ := rootDecoder.Token()
key.DisableQuery, _ = strconv.ParseBool(fmt.Sprint(val))
case "disable_scheme":
val, _ := rootDecoder.Token()
key.DisableScheme, _ = strconv.ParseBool(fmt.Sprint(val))
case "hash":
val, _ := rootDecoder.Token()
key.Hash, _ = strconv.ParseBool(fmt.Sprint(val))
case "hide":
val, _ := rootDecoder.Token()
key.Hide, _ = strconv.ParseBool(fmt.Sprint(val))
Expand Down Expand Up @@ -208,6 +214,8 @@ type Key struct {
DisableHost bool `json:"disable_host,omitempty" yaml:"disable_host,omitempty"`
DisableMethod bool `json:"disable_method,omitempty" yaml:"disable_method,omitempty"`
DisableQuery bool `json:"disable_query,omitempty" yaml:"disable_query,omitempty"`
DisableScheme bool `json:"disable_scheme,omitempty" yaml:"disable_scheme,omitempty"`
Hash bool `json:"hash,omitempty" yaml:"hash,omitempty"`
Hide bool `json:"hide,omitempty" yaml:"hide,omitempty"`
Headers []string `json:"headers,omitempty" yaml:"headers,omitempty"`
}
Expand Down
42 changes: 35 additions & 7 deletions context/key.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,17 @@ const (
Key ctxKey = "souin_ctx.CACHE_KEY"
DisplayableKey ctxKey = "souin_ctx.DISPLAYABLE_KEY"
IgnoredHeaders ctxKey = "souin_ctx.IGNORE_HEADERS"
Hashed ctxKey = "souin_ctx.HASHED"
)

type keyContext struct {
disable_body bool
disable_host bool
disable_method bool
disable_query bool
disable_scheme bool
displayable bool
hash bool
headers []string
overrides []map[*regexp.Regexp]keyContext
}
Expand All @@ -34,6 +37,8 @@ func (g *keyContext) SetupContext(c configurationtypes.AbstractConfigurationInte
g.disable_host = k.DisableHost
g.disable_method = k.DisableMethod
g.disable_query = k.DisableQuery
g.disable_scheme = k.DisableScheme
g.hash = k.Hash
g.displayable = !k.Hide
g.headers = k.Headers

Expand All @@ -46,6 +51,8 @@ func (g *keyContext) SetupContext(c configurationtypes.AbstractConfigurationInte
disable_host: v.DisableHost,
disable_method: v.DisableMethod,
disable_query: v.DisableQuery,
disable_scheme: v.DisableScheme,
hash: v.Hash,
displayable: !v.Hide,
headers: v.Headers,
}})
Expand All @@ -57,11 +64,9 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
key := req.URL.Path
var headers []string

scheme := "http-"
if req.TLS != nil {
scheme = "https-"
}
hash := g.hash
query := ""
scheme := ""
body := ""
host := ""
method := ""
Expand All @@ -80,6 +85,13 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
host = req.Host + "-"
}

if !g.disable_scheme {
scheme = "http-"
if req.TLS != nil {
scheme = "https-"
}
}

if !g.disable_method {
method = req.Method + "-"
}
Expand All @@ -97,6 +109,7 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
host = ""
method = ""
query = ""
scheme = ""
if !v.disable_query && len(req.URL.RawQuery) > 0 {
query = "?" + req.URL.RawQuery
}
Expand All @@ -109,13 +122,22 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
if !v.disable_host {
host = req.Host + "-"
}
if !v.disable_scheme {
scheme = "http-"
if req.TLS != nil {
scheme = "https-"
}
}
if len(v.headers) > 0 {
headerValues = ""
for _, hn := range v.headers {
headers = v.headers
headerValues += "-" + req.Header.Get(hn)
}
}
if v.hash {
hash = true
}
hasOverride = true
break
}
Expand All @@ -126,13 +148,19 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
}
}

key = method + scheme + host + key + query + body + headerValues

return req.WithContext(
context.WithValue(
context.WithValue(
context.WithValue(
req.Context(),
Key,
method+scheme+host+key+query+body+headerValues,
context.WithValue(
req.Context(),
Key,
key,
),
Hashed,
hash,
),
IgnoredHeaders,
headers,
Expand Down
Loading
Loading