174 lines
7.0 KiB
C
174 lines
7.0 KiB
C
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-│
|
|
│vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi│
|
|
╞══════════════════════════════════════════════════════════════════════════════╡
|
|
│ Copyright 2013 Intel Corporation │
|
|
│ Use of this source code is governed by the BSD-style licenses that can │
|
|
│ be found in the third_party/zlib/LICENSE file. │
|
|
╚─────────────────────────────────────────────────────────────────────────────*/
|
|
#include "libc/bits/emmintrin.h"
|
|
#include "libc/str/str.h"
|
|
#include "third_party/zlib/deflate.h"
|
|
#include "third_party/zlib/internal.h"
|
|
#include "third_party/zlib/zutil.h"
|
|
|
|
asm(".ident\t\"\\n\\n\
|
|
zlib » sse2 fill window (zlib License)\\n\
|
|
Copyright 2013 Intel Corporation\\n\
|
|
Authors: Arjan van de Ven, Jim Kukunas\"");
|
|
asm(".include \"libc/disclaimer.inc\"");
|
|
|
|
/**
|
|
* @fileoverview Fill Window with SSE2-optimized hash shifting
|
|
*/
|
|
|
|
#define UPDATE_HASH(s, h, i) \
|
|
{ \
|
|
if (s->level < 6) { \
|
|
h = (3483 * (s->window[i]) + 23081 * (s->window[i + 1]) + \
|
|
6954 * (s->window[i + 2]) + 20947 * (s->window[i + 3])) & \
|
|
s->hash_mask; \
|
|
} else { \
|
|
h = (25881 * (s->window[i]) + 24674 * (s->window[i + 1]) + \
|
|
25811 * (s->window[i + 2])) & \
|
|
s->hash_mask; \
|
|
} \
|
|
}
|
|
|
|
void fill_window_sse(struct DeflateState *s) {
|
|
const __m128i xmm_wsize = _mm_set1_epi16(s->w_size);
|
|
|
|
register unsigned n;
|
|
register Posf *p;
|
|
unsigned more; /* Amount of free space at the end of the window. */
|
|
uInt wsize = s->w_size;
|
|
|
|
Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead");
|
|
|
|
do {
|
|
more = (unsigned)(s->window_size - (uint64_t)s->lookahead -
|
|
(uint64_t)s->strstart);
|
|
|
|
/* Deal with !@#$% 64K limit: */
|
|
if (sizeof(int) <= 2) {
|
|
if (more == 0 && s->strstart == 0 && s->lookahead == 0) {
|
|
more = wsize;
|
|
|
|
} else if (more == (unsigned)(-1)) {
|
|
/* Very unlikely, but possible on 16 bit machine if
|
|
* strstart == 0 && lookahead == 1 (input done a byte at time)
|
|
*/
|
|
more--;
|
|
}
|
|
}
|
|
|
|
/* If the window is almost full and there is insufficient lookahead,
|
|
* move the upper half to the lower one to make room in the upper half.
|
|
*/
|
|
if (s->strstart >= wsize + MAX_DIST(s)) {
|
|
memcpy(s->window, s->window + wsize, (unsigned)wsize);
|
|
s->match_start -= wsize;
|
|
s->strstart -= wsize; /* we now have strstart >= MAX_DIST */
|
|
s->block_start -= (long)wsize;
|
|
|
|
/* Slide the hash table (could be avoided with 32 bit values
|
|
at the expense of memory usage). We slide even when level == 0
|
|
to keep the hash table consistent if we switch back to level > 0
|
|
later. (Using level 0 permanently is not an optimal usage of
|
|
zlib, so we don't care about this pathological case.)
|
|
*/
|
|
n = s->hash_size;
|
|
p = &s->head[n];
|
|
p -= 8;
|
|
do {
|
|
__m128i value, result;
|
|
value = _mm_loadu_si128((__m128i *)p);
|
|
result = _mm_subs_epu16(value, xmm_wsize);
|
|
_mm_storeu_si128((__m128i *)p, result);
|
|
p -= 8;
|
|
n -= 8;
|
|
} while (n > 0);
|
|
|
|
n = wsize;
|
|
#ifndef FASTEST
|
|
p = &s->prev[n];
|
|
p -= 8;
|
|
do {
|
|
__m128i value, result;
|
|
value = _mm_loadu_si128((__m128i *)p);
|
|
result = _mm_subs_epu16(value, xmm_wsize);
|
|
_mm_storeu_si128((__m128i *)p, result);
|
|
p -= 8;
|
|
n -= 8;
|
|
} while (n > 0);
|
|
#endif
|
|
more += wsize;
|
|
}
|
|
if (s->strm->avail_in == 0) break;
|
|
|
|
/* If there was no sliding:
|
|
* strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 &&
|
|
* more == window_size - lookahead - strstart
|
|
* => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1)
|
|
* => more >= window_size - 2*WSIZE + 2
|
|
* In the BIG_MEM or MMAP case (not yet supported),
|
|
* window_size == input_size + MIN_LOOKAHEAD &&
|
|
* strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD.
|
|
* Otherwise, window_size == 2*WSIZE so more >= 2.
|
|
* If there was sliding, more >= WSIZE. So in all cases, more >= 2.
|
|
*/
|
|
Assert(more >= 2, "more < 2");
|
|
|
|
n = deflate_read_buf(s->strm, s->window + s->strstart + s->lookahead, more);
|
|
s->lookahead += n;
|
|
|
|
/* Initialize the hash value now that we have some input: */
|
|
if (s->lookahead >= MIN_MATCH) {
|
|
uInt str = s->strstart;
|
|
s->ins_h = s->window[str];
|
|
if (str >= 1) UPDATE_HASH(s, s->ins_h, str + 1 - (MIN_MATCH - 1));
|
|
#if MIN_MATCH != 3
|
|
Call UPDATE_HASH() MIN_MATCH - 3 more times
|
|
#endif
|
|
}
|
|
/* If the whole input has less than MIN_MATCH bytes, ins_h is garbage,
|
|
* but this is not important since only literal bytes will be emitted.
|
|
*/
|
|
|
|
} while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0);
|
|
|
|
/* If the WIN_INIT bytes after the end of the current data have never been
|
|
* written, then zero those bytes in order to avoid memory check reports of
|
|
* the use of uninitialized (or uninitialised as Julian writes) bytes by
|
|
* the longest match routines. Update the high water mark for the next
|
|
* time through here. WIN_INIT is set to MAX_MATCH since the longest match
|
|
* routines allow scanning to strstart + MAX_MATCH, ignoring lookahead.
|
|
*/
|
|
if (s->high_water < s->window_size) {
|
|
uint64_t curr = s->strstart + (uint64_t)(s->lookahead);
|
|
uint64_t init;
|
|
|
|
if (s->high_water < curr) {
|
|
/* Previous high water mark below current data -- zero WIN_INIT
|
|
* bytes or up to end of window, whichever is less.
|
|
*/
|
|
init = s->window_size - curr;
|
|
if (init > WIN_INIT) init = WIN_INIT;
|
|
memset(s->window + curr, 0, init);
|
|
s->high_water = curr + init;
|
|
} else if (s->high_water < (uint64_t)curr + WIN_INIT) {
|
|
/* High water mark at or above current data, but below current data
|
|
* plus WIN_INIT -- zero out to current data plus WIN_INIT, or up
|
|
* to end of window, whichever is less.
|
|
*/
|
|
init = (uint64_t)curr + WIN_INIT - s->high_water;
|
|
if (init > s->window_size - s->high_water)
|
|
init = s->window_size - s->high_water;
|
|
memset(s->window + s->high_water, 0, init);
|
|
s->high_water += init;
|
|
}
|
|
}
|
|
|
|
Assert((uint64_t)s->strstart <= s->window_size - MIN_LOOKAHEAD,
|
|
"not enough room for search");
|
|
}
|